From 5fb3be7a708462bd28482a7f734d706cb9fb688f Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Fri, 2 Oct 2026 17:51:01 +0100 Subject: [PATCH 01/16] Add OpenRouter plugin Imports models, workspaces, and API keys, with usage, model, benchmark, and ranking streams, default dashboards, and setup docs. Squashed from: - Add OpenRouter plugin shell - Import OpenRouter models, workspaces, and API keys - Add OpenRouter usage, model, benchmark, and ranking streams - Use date shaping instead of code for OpenRouter dates - Add OpenRouter dashboards and custom types - Add per-model benchmarks radar and drop context length from comparison radar - Finish OpenRouter README Co-Authored-By: Claude Opus 5.5 --- plugins/OpenRouter/v1/configValidation.json | 22 + plugins/OpenRouter/v1/custom_types.json | 23 + .../OpenRouter/v1/dataStreams/apiKeys.json | 98 +++ .../OpenRouter/v1/dataStreams/benchmarks.json | 114 ++++ .../OpenRouter/v1/dataStreams/credits.json | 38 ++ .../v1/dataStreams/modelProviders.json | 138 ++++ plugins/OpenRouter/v1/dataStreams/models.json | 204 ++++++ .../OpenRouter/v1/dataStreams/rankings.json | 159 +++++ .../v1/dataStreams/scripts/benchmarks.js | 53 ++ .../v1/dataStreams/scripts/models.js | 9 + .../v1/dataStreams/scripts/usage.js | 28 + plugins/OpenRouter/v1/dataStreams/usage.json | 151 +++++ .../OpenRouter/v1/dataStreams/workspaces.json | 44 ++ .../v1/defaultContent/apiKey.dash.json | 343 ++++++++++ .../v1/defaultContent/manifest.json | 24 + .../v1/defaultContent/model.dash.json | 593 ++++++++++++++++ .../defaultContent/modelComparison.dash.json | 631 ++++++++++++++++++ .../v1/defaultContent/overview.dash.json | 574 ++++++++++++++++ .../OpenRouter/v1/defaultContent/scopes.json | 53 ++ .../v1/defaultContent/workspace.dash.json | 468 +++++++++++++ plugins/OpenRouter/v1/docs/README.md | 73 ++ plugins/OpenRouter/v1/icon.svg | 6 + .../v1/indexDefinitions/default.json | 55 ++ plugins/OpenRouter/v1/metadata.json | 54 ++ plugins/OpenRouter/v1/ui.json | 12 + 25 files changed, 3967 insertions(+) create mode 100644 plugins/OpenRouter/v1/configValidation.json create mode 100644 plugins/OpenRouter/v1/custom_types.json create mode 100644 plugins/OpenRouter/v1/dataStreams/apiKeys.json create mode 100644 plugins/OpenRouter/v1/dataStreams/benchmarks.json create mode 100644 plugins/OpenRouter/v1/dataStreams/credits.json create mode 100644 plugins/OpenRouter/v1/dataStreams/modelProviders.json create mode 100644 plugins/OpenRouter/v1/dataStreams/models.json create mode 100644 plugins/OpenRouter/v1/dataStreams/rankings.json create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/benchmarks.js create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/models.js create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/usage.js create mode 100644 plugins/OpenRouter/v1/dataStreams/usage.json create mode 100644 plugins/OpenRouter/v1/dataStreams/workspaces.json create mode 100644 plugins/OpenRouter/v1/defaultContent/apiKey.dash.json create mode 100644 plugins/OpenRouter/v1/defaultContent/manifest.json create mode 100644 plugins/OpenRouter/v1/defaultContent/model.dash.json create mode 100644 plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json create mode 100644 plugins/OpenRouter/v1/defaultContent/overview.dash.json create mode 100644 plugins/OpenRouter/v1/defaultContent/scopes.json create mode 100644 plugins/OpenRouter/v1/defaultContent/workspace.dash.json create mode 100644 plugins/OpenRouter/v1/docs/README.md create mode 100644 plugins/OpenRouter/v1/icon.svg create mode 100644 plugins/OpenRouter/v1/indexDefinitions/default.json create mode 100644 plugins/OpenRouter/v1/metadata.json create mode 100644 plugins/OpenRouter/v1/ui.json diff --git a/plugins/OpenRouter/v1/configValidation.json b/plugins/OpenRouter/v1/configValidation.json new file mode 100644 index 00000000..eb07eb2b --- /dev/null +++ b/plugins/OpenRouter/v1/configValidation.json @@ -0,0 +1,22 @@ +{ + "steps": [ + { + "displayName": "Authenticate", + "dataStream": { + "name": "credits" + }, + "required": true, + "success": "Connected to OpenRouter.", + "error": "Could not authenticate. Check that the key is valid, enabled, and has not expired." + }, + { + "displayName": "Check management access", + "dataStream": { + "name": "workspaces" + }, + "required": false, + "success": "Management key detected. All data streams are available.", + "error": "Could not read workspaces. Workspaces, API keys, and usage need a management key; model, benchmark, ranking, and credit data still work." + } + ] +} diff --git a/plugins/OpenRouter/v1/custom_types.json b/plugins/OpenRouter/v1/custom_types.json new file mode 100644 index 00000000..72926835 --- /dev/null +++ b/plugins/OpenRouter/v1/custom_types.json @@ -0,0 +1,23 @@ +[ + { + "name": "Model", + "sourceType": "Model", + "icon": "brain", + "singular": "Model", + "plural": "Models" + }, + { + "name": "Workspace", + "sourceType": "Workspace", + "icon": "layer-group", + "singular": "Workspace", + "plural": "Workspaces" + }, + { + "name": "API Key", + "sourceType": "API Key", + "icon": "key", + "singular": "API Key", + "plural": "API Keys" + } +] diff --git a/plugins/OpenRouter/v1/dataStreams/apiKeys.json b/plugins/OpenRouter/v1/dataStreams/apiKeys.json new file mode 100644 index 00000000..4496bf21 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/apiKeys.json @@ -0,0 +1,98 @@ +{ + "name": "apiKeys", + "displayName": "API Keys", + "description": "API keys in a workspace, with spend and limits in USD (requires a management key)", + "tags": ["API Keys", "Cost"], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "get", + "endpointPath": "keys", + "paging": { + "mode": "offset", + "offset": { + "mode": "row", + "rowCountIn": { "realm": "payloadArraySize", "path": "data" }, + "base": 0 + }, + "out": { "realm": "queryArg", "path": "offset" } + }, + "getArgs": [ + { "key": "workspace_id", "value": "{{object.rawId}}" }, + { "key": "include_disabled", "value": "true" } + ], + "headers": [], + "pathToData": "data" + }, + "matches": { + "sourceType": { "type": "oneOf", "values": ["Workspace"] } + }, + "metadata": [ + { + "name": "hash", + "displayName": "Key Hash", + "shape": "string" + }, + { + "name": "name", + "displayName": "Name", + "shape": "string", + "role": "label" + }, + { + "name": "disabled", + "displayName": "Disabled", + "shape": "boolean" + }, + { + "name": "usage_daily", + "displayName": "Spend Today", + "shape": "usd" + }, + { + "name": "usage_weekly", + "displayName": "Spend This Week", + "shape": "usd" + }, + { + "name": "usage_monthly", + "displayName": "Spend This Month", + "shape": "usd" + }, + { + "name": "usage", + "displayName": "Total Spend", + "shape": "usd" + }, + { + "name": "limit", + "displayName": "Limit", + "shape": "usd" + }, + { + "name": "limit_remaining", + "displayName": "Limit Remaining", + "shape": "usd" + }, + { + "name": "limit_reset", + "displayName": "Limit Reset", + "shape": "string" + }, + { + "name": "workspace_id", + "displayName": "Workspace ID", + "shape": "string" + }, + { + "name": "created_at", + "displayName": "Created", + "shape": "date" + }, + { + "name": "expires_at", + "displayName": "Expires", + "shape": "date" + } + ], + "timeframes": false +} diff --git a/plugins/OpenRouter/v1/dataStreams/benchmarks.json b/plugins/OpenRouter/v1/dataStreams/benchmarks.json new file mode 100644 index 00000000..8498d983 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/benchmarks.json @@ -0,0 +1,114 @@ +{ + "name": "benchmarks", + "displayName": "Benchmarks", + "description": "Model benchmark scores from Artificial Analysis, Design Arena, and OpenRouter evals (refreshed daily)", + "tags": [ + "Models", + "Benchmarks" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "benchmarks", + "paging": { + "mode": "none" + }, + "getArgs": [ + { + "key": "source", + "value": "{{source || null}}" + } + ], + "headers": [], + "postRequestScript": "benchmarks.js" + }, + "matches": "none", + "ui": [ + { + "type": "choiceChips", + "name": "source", + "label": "Source", + "options": [ + { + "value": "artificial-analysis", + "label": "Artificial Analysis" + }, + { + "value": "design-arena", + "label": "Design Arena" + }, + { + "value": "openrouter", + "label": "OpenRouter evals" + } + ] + } + ], + "metadata": [ + { + "name": "model", + "displayName": "Model", + "shape": "string" + }, + { + "name": "modelName", + "displayName": "Model Name", + "shape": "string", + "role": "label" + }, + { + "name": "source", + "displayName": "Source", + "shape": "string" + }, + { + "name": "benchmark", + "displayName": "Benchmark", + "shape": "string" + }, + { + "name": "metric", + "displayName": "Metric", + "shape": "string" + }, + { + "name": "score", + "displayName": "Score", + "shape": [ + "number", + { + "decimalPlaces": 2 + } + ] + }, + { + "name": "winRate", + "displayName": "Win Rate", + "shape": "percent" + }, + { + "name": "tasks", + "displayName": "Tasks", + "shape": "number" + }, + { + "name": "avgCostPerTask", + "displayName": "Avg Cost per Task", + "shape": "usd" + }, + { + "name": "avgDurationMs", + "displayName": "Avg Duration", + "shape": "milliseconds" + }, + { + "name": "lastRun", + "displayName": "Last Run", + "shape": "date" + } + ], + "options": { + "minimumRequestIntervalSeconds": 86400 + }, + "timeframes": false +} diff --git a/plugins/OpenRouter/v1/dataStreams/credits.json b/plugins/OpenRouter/v1/dataStreams/credits.json new file mode 100644 index 00000000..26debe83 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/credits.json @@ -0,0 +1,38 @@ +{ + "name": "credits", + "displayName": "Credits", + "description": "Credits purchased, used, and remaining for the account (USD)", + "tags": ["Cost", "Billing"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "credits", + "paging": { + "mode": "none" + }, + "getArgs": [], + "headers": [], + "pathToData": "data" + }, + "matches": "none", + "metadata": [ + { + "name": "total_credits", + "displayName": "Total Credits", + "shape": "usd" + }, + { + "name": "total_usage", + "displayName": "Total Usage", + "shape": "usd" + }, + { + "name": "remaining", + "displayName": "Remaining", + "shape": "usd", + "computed": true, + "valueExpression": "{{ $['total_credits'] - $['total_usage'] }}" + } + ], + "timeframes": false +} diff --git a/plugins/OpenRouter/v1/dataStreams/modelProviders.json b/plugins/OpenRouter/v1/dataStreams/modelProviders.json new file mode 100644 index 00000000..ad0d00c8 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/modelProviders.json @@ -0,0 +1,138 @@ +{ + "name": "modelProviders", + "displayName": "Model Providers", + "description": "Providers serving a model, with price, uptime, latency, and throughput (last 30 minutes)", + "tags": [ + "Models", + "Performance" + ], + "baseDataSourceName": "httpRequestScopedSingle", + "config": { + "httpMethod": "get", + "endpointPath": "models/{{object.rawId}}/endpoints", + "paging": { + "mode": "none" + }, + "getArgs": [], + "headers": [], + "pathToData": "data.endpoints", + "expandInnerObjects": true + }, + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Model" + ] + } + }, + "metadata": [ + { + "name": "model_name", + "displayName": "Model", + "shape": "string" + }, + { + "name": "provider_name", + "displayName": "Provider", + "shape": "string", + "role": "label" + }, + { + "name": "tag", + "displayName": "Tag", + "shape": "string" + }, + { + "name": "quantization", + "displayName": "Quantization", + "shape": "string" + }, + { + "name": "context_length", + "displayName": "Context Length", + "shape": "number" + }, + { + "name": "pricing.prompt", + "visible": false, + "displayName": "Input Price (per Token)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 8 + } + ] + }, + { + "name": "inputPrice", + "displayName": "Input Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ], + "computed": true, + "valueExpression": "{{ (v => v == null || v === '' || Number(v) < 0 ? null : Number(v) * 1e6)($['pricing.prompt']) }}" + }, + { + "name": "pricing.completion", + "visible": false, + "displayName": "Output Price (per Token)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 8 + } + ] + }, + { + "name": "outputPrice", + "displayName": "Output Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ], + "computed": true, + "valueExpression": "{{ (v => v == null || v === '' || Number(v) < 0 ? null : Number(v) * 1e6)($['pricing.completion']) }}" + }, + { + "name": "uptime_last_30m", + "displayName": "Uptime (30m)", + "shape": "percent" + }, + { + "name": "uptime_last_1d", + "displayName": "Uptime (1d)", + "shape": "percent" + }, + { + "name": "latency_last_30m.p50", + "displayName": "Latency p50", + "shape": "milliseconds" + }, + { + "name": "latency_last_30m.p90", + "displayName": "Latency p90", + "shape": "milliseconds" + }, + { + "name": "throughput_last_30m.p50", + "displayName": "Throughput p50 (Tokens/s)", + "shape": [ + "number", + { + "decimalPlaces": 1 + } + ] + } + ], + "timeframes": false +} diff --git a/plugins/OpenRouter/v1/dataStreams/models.json b/plugins/OpenRouter/v1/dataStreams/models.json new file mode 100644 index 00000000..7095a89e --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/models.json @@ -0,0 +1,204 @@ +{ + "name": "models", + "displayName": "Models", + "description": "Models available through OpenRouter, with pricing, context length, and benchmark scores", + "tags": [ + "Models" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "models", + "paging": { + "mode": "none" + }, + "getArgs": [], + "headers": [], + "expandInnerObjects": true, + "postRequestScript": "models.js" + }, + "matches": "none", + "metadata": [ + { + "name": "id", + "displayName": "Model ID", + "shape": "string" + }, + { + "name": "name", + "displayName": "Name", + "shape": "string", + "role": "label" + }, + { + "name": "author", + "displayName": "Author", + "shape": "string", + "computed": true, + "valueExpression": "{{ $['id'].split('/')[0] }}" + }, + { + "name": "canonical_slug", + "displayName": "Canonical Slug", + "shape": "string" + }, + { + "name": "context_length", + "displayName": "Context Length", + "shape": "number" + }, + { + "name": "inputPrice", + "displayName": "Input Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ], + "computed": true, + "valueExpression": "{{ (v => v == null || v === '' || Number(v) < 0 ? null : Number(v) * 1e6)($['pricing.prompt']) }}" + }, + { + "name": "outputPrice", + "displayName": "Output Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ], + "computed": true, + "valueExpression": "{{ (v => v == null || v === '' || Number(v) < 0 ? null : Number(v) * 1e6)($['pricing.completion']) }}" + }, + { + "name": "cacheReadPrice", + "displayName": "Cache Read Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ], + "computed": true, + "valueExpression": "{{ (v => v == null || v === '' || Number(v) < 0 ? null : Number(v) * 1e6)($['pricing.input_cache_read']) }}" + }, + { + "name": "top_provider.max_completion_tokens", + "displayName": "Max Completion Tokens", + "shape": "number" + }, + { + "name": "inputModalities", + "displayName": "Input Modalities", + "shape": "string", + "computed": true, + "valueExpression": "{{ JSON.parse($['architecture.input_modalities'] || '[]').join(', ') }}" + }, + { + "name": "outputModalities", + "displayName": "Output Modalities", + "shape": "string", + "computed": true, + "valueExpression": "{{ JSON.parse($['architecture.output_modalities'] || '[]').join(', ') }}" + }, + { + "name": "benchmarks.artificial_analysis.intelligence_index", + "displayName": "Intelligence Index", + "shape": "number" + }, + { + "name": "benchmarks.artificial_analysis.coding_index", + "displayName": "Coding Index", + "shape": "number" + }, + { + "name": "benchmarks.artificial_analysis.agentic_index", + "displayName": "Agentic Index", + "shape": "number" + }, + { + "name": "knowledge_cutoff", + "displayName": "Knowledge Cutoff", + "shape": "string" + }, + { + "name": "created", + "displayName": "Created", + "shape": [ + "date", + { + "inputPattern": "t" + } + ] + }, + { + "name": "expiration_date", + "displayName": "Expiration Date", + "shape": "string" + }, + { + "name": "pricing.prompt", + "visible": false, + "displayName": "Input Price (per Token)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 8 + } + ] + }, + { + "name": "pricing.completion", + "visible": false, + "displayName": "Output Price (per Token)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 8 + } + ] + }, + { + "name": "pricing.input_cache_read", + "visible": false, + "displayName": "Cache Read Price (per Token)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 8 + } + ] + }, + { + "name": "architecture.input_modalities", + "visible": false + }, + { + "name": "architecture.output_modalities", + "visible": false + } + ], + "timeframes": false, + "ui": [ + { + "type": "objects", + "name": "models", + "label": "Model(s)", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Model" + ] + } + } + } + ] +} diff --git a/plugins/OpenRouter/v1/dataStreams/rankings.json b/plugins/OpenRouter/v1/dataStreams/rankings.json new file mode 100644 index 00000000..e238fa0b --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/rankings.json @@ -0,0 +1,159 @@ +{ + "name": "rankings", + "displayName": "Model Rankings", + "description": "Top models by tokens processed across all OpenRouter traffic (daily, from OpenRouter's public rankings dataset)", + "tags": [ + "Models", + "Usage" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "datasets/rankings-daily", + "paging": { + "mode": "none" + }, + "getArgs": [ + { + "key": "start_date", + "value": "{{timeframe.start.substring(0, 10)}}" + }, + { + "key": "end_date", + "value": "{{timeframe.end.substring(0, 10)}}" + }, + { + "key": "period", + "value": "{{period || null}}" + }, + { + "key": "category", + "value": "{{category || null}}" + } + ], + "headers": [], + "pathToData": "data" + }, + "matches": "none", + "ui": [ + { + "type": "choiceChips", + "name": "period", + "label": "Period", + "options": [ + { + "value": "day", + "label": "Day" + }, + { + "value": "week", + "label": "Week" + }, + { + "value": "month", + "label": "Month" + } + ] + }, + { + "type": "choiceChips", + "name": "category", + "label": "Category", + "options": [ + { + "value": "programming", + "label": "Programming" + }, + { + "value": "roleplay", + "label": "Roleplay" + }, + { + "value": "marketing", + "label": "Marketing" + }, + { + "value": "marketing/seo", + "label": "Marketing / SEO" + }, + { + "value": "technology", + "label": "Technology" + }, + { + "value": "science", + "label": "Science" + }, + { + "value": "translation", + "label": "Translation" + }, + { + "value": "legal", + "label": "Legal" + }, + { + "value": "finance", + "label": "Finance" + }, + { + "value": "health", + "label": "Health" + }, + { + "value": "trivia", + "label": "Trivia" + }, + { + "value": "academia", + "label": "Academia" + } + ] + } + ], + "metadata": [ + { + "name": "date", + "displayName": "Date", + "shape": "date", + "role": "timestamp" + }, + { + "name": "model_permaslug", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "author", + "displayName": "Author", + "shape": "string", + "computed": true, + "valueExpression": "{{ $['model_permaslug']?.split('/')[0] }}" + }, + { + "name": "total_tokens", + "displayName": "Total Tokens", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + } + ], + "timeframes": [ + "last7days", + "last30days", + "thisMonth", + "lastMonth", + "thisQuarter", + "lastQuarter", + "thisYear", + "lastYear" + ], + "options": { + "minimumRequestIntervalSeconds": 86400 + } +} diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/benchmarks.js b/plugins/OpenRouter/v1/dataStreams/scripts/benchmarks.js new file mode 100644 index 00000000..1b4862e1 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/benchmarks.js @@ -0,0 +1,53 @@ +// Flatten each source's differently shaped rows into one row per model x benchmark +const aaIndices = [ + ['intelligence_index', 'Intelligence Index'], + ['coding_index', 'Coding Index'], + ['agentic_index', 'Agentic Index'], +]; + +const rows = []; + +for (const r of data?.data ?? []) { + const base = { + source: r.source, + model: r.model_permaslug, + modelName: r.display_name, + }; + + if (r.source === 'artificial-analysis') { + for (const [key, benchmark] of aaIndices) { + if (r[key] == null) continue; + + rows.push({ ...base, benchmark, metric: 'index', score: r[key] }); + } + } else if (r.source === 'design-arena') { + if (r.elo == null) continue; + + rows.push({ + ...base, + benchmark: r.category, + metric: 'elo', + score: r.elo, + winRate: r.win_rate, + tasks: r.tournament_stats?.total, + avgDurationMs: r.avg_generation_time_ms, + }); + } else if (r.source === 'openrouter') { + // Search benchmarks report primary_score instead of accuracy + const score = r.accuracy ?? r.primary_score; + if (score == null) continue; + + rows.push({ + ...base, + benchmark: r.benchmark_type, + metric: r.accuracy != null ? 'accuracy' : r.primary_metric, + score, + tasks: r.total_tasks, + avgCostPerTask: r.avg_cost_per_task, + avgDurationMs: r.avg_latency_per_task_ms, + lastRun: r.last_run_timestamp, + }); + } +} + +result = rows; diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/models.js b/plugins/OpenRouter/v1/dataStreams/scripts/models.js new file mode 100644 index 00000000..265a97d9 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/models.js @@ -0,0 +1,9 @@ +// Optional `models` object-picker parameter: selected objects arrive at +// context.config.models as an array, each rawId a single-element array. +// Empty/absent (including the import run) returns every model. +const unwrap = (v) => (Array.isArray(v) ? v[0] : v); +const selected = context.config?.models ?? []; +const ids = new Set(selected.map((o) => unwrap(o.rawId)).filter(Boolean)); + +const models = data?.data ?? []; +result = ids.size ? models.filter((m) => ids.has(m.id)) : models; diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/usage.js b/plugins/OpenRouter/v1/dataStreams/scripts/usage.js new file mode 100644 index 00000000..6e797450 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/usage.js @@ -0,0 +1,28 @@ +// Normalises analytics rows: the time key is named after the granularity +// (date__day, date__hour, ...) and large counts arrive as strings. +const body = data?.data ?? {}; +const metric = context.config.metric || 'total_usage'; +const groupBy = context.config.groupBy; + +if (body.metadata?.truncated) { + api.report.warning( + 'Results were truncated by OpenRouter; narrow the timeframe or filter to specific objects', + ); +} + +const dateKey = (row) => Object.keys(row).find((k) => k.startsWith('date__')); + +result = (body.data ?? []) + .map((row) => { + const date = row[dateKey(row)]; + if (!date) return null; + + const value = row[metric]; + + return { + date, + group: groupBy ? row[groupBy] : null, + value: value == null ? null : Number(value), + }; + }) + .filter(Boolean); diff --git a/plugins/OpenRouter/v1/dataStreams/usage.json b/plugins/OpenRouter/v1/dataStreams/usage.json new file mode 100644 index 00000000..460cc43c --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/usage.json @@ -0,0 +1,151 @@ +{ + "name": "usage", + "displayName": "Usage", + "description": "Spend, requests, and tokens over time, optionally grouped by model, provider, workspace, API key, or app (requires a management key)", + "tags": [ + "Usage", + "Cost" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "analytics/query", + "paging": { + "mode": "none" + }, + "getArgs": [], + "headers": [], + "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; const days = (new Date(timeframe.end) - new Date(timeframe.start)) / 864e5; const granularity = days <= 2 ? 'hour' : days <= 31 ? 'day' : days <= 92 ? 'week' : 'month'; const selected = typeof filterObjects !== 'undefined' && Array.isArray(filterObjects) ? filterObjects : []; const one = (v) => (Array.isArray(v) ? v[0] : v); const idsOf = (type, ...props) => [...new Set(selected.filter((o) => one(o.sourceType) === type).flatMap((o) => props.map((p) => one(o[p]))).filter(Boolean))]; const rules = [ ['model', idsOf('Model', 'rawId', 'canonical_slug')], ['workspace', idsOf('Workspace', 'rawId')], ['api_key_id', idsOf('API Key', 'rawId')] ]; const filters = rules.filter(([, values]) => values.length).map(([field, value]) => ({ field, operator: 'in', value })); return JSON.stringify({ metrics: [typeof metric !== 'undefined' && metric ? metric : 'total_usage'], dimensions: typeof groupBy !== 'undefined' && groupBy ? [groupBy] : [], filters, granularity, time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", + "postRequestScript": "usage.js" + }, + "matches": "none", + "ui": [ + { + "type": "choiceChips", + "name": "metric", + "label": "Metric", + "defaultValue": "total_usage", + "validation": { + "required": true + }, + "options": [ + { + "value": "total_usage", + "label": "Spend (USD)" + }, + { + "value": "request_count", + "label": "Requests" + }, + { + "value": "tokens_total", + "label": "Total tokens" + }, + { + "value": "tokens_prompt", + "label": "Prompt tokens" + }, + { + "value": "tokens_completion", + "label": "Completion tokens" + }, + { + "value": "reasoning_tokens", + "label": "Reasoning tokens" + }, + { + "value": "cache_hit_rate", + "label": "Cache hit rate" + } + ] + }, + { + "type": "choiceChips", + "name": "groupBy", + "label": "Group by", + "options": [ + { + "value": "model", + "label": "Model" + }, + { + "value": "provider", + "label": "Provider (31 days max)" + }, + { + "value": "workspace", + "label": "Workspace" + }, + { + "value": "api_key_id", + "label": "API key" + }, + { + "value": "app", + "label": "App" + } + ] + }, + { + "type": "objects", + "name": "filterObjects", + "label": "Filter to objects (optional)", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Model", + "Workspace", + "API Key" + ] + } + } + } + ], + "metadata": [ + { + "name": "date", + "displayName": "Date", + "shape": "date", + "role": "timestamp" + }, + { + "name": "group", + "displayName": "Group", + "shape": "string", + "role": "label" + }, + { + "name": "value", + "displayName": "Value", + "shape": [ + "number", + { + "decimalPlaces": 4 + } + ], + "role": "value" + } + ], + "timeframes": [ + "last12hours", + "last24hours", + "last7days", + "last30days", + "thisMonth", + "lastMonth", + "thisQuarter", + "lastQuarter", + "thisYear" + ], + "defaultShaping": { + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + } + } +} diff --git a/plugins/OpenRouter/v1/dataStreams/workspaces.json b/plugins/OpenRouter/v1/dataStreams/workspaces.json new file mode 100644 index 00000000..b0813319 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/workspaces.json @@ -0,0 +1,44 @@ +{ + "name": "workspaces", + "displayName": "Workspaces", + "description": "Workspaces in the OpenRouter account (requires a management key)", + "tags": ["Workspaces"], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "workspaces", + "paging": { + "mode": "offset", + "pageSize": { "realm": "queryArg", "path": "limit", "value": "100" }, + "offset": { + "mode": "row", + "rowCountIn": { "realm": "payloadArraySize", "path": "data" }, + "base": 0 + }, + "out": { "realm": "queryArg", "path": "offset" } + }, + "getArgs": [], + "headers": [], + "pathToData": "data" + }, + "matches": "none", + "metadata": [ + { + "name": "id", + "displayName": "Workspace ID", + "shape": "string" + }, + { + "name": "name", + "displayName": "Name", + "shape": "string", + "role": "label" + }, + { + "name": "slug", + "displayName": "Slug", + "shape": "string" + } + ], + "timeframes": false +} diff --git a/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json new file mode 100644 index 00000000..b25cbd10 --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json @@ -0,0 +1,343 @@ +{ + "name": "API Key", + "schemaVersion": "1.5", + "timeframe": "last30days", + "variables": [ + "{{variables.[API Key]}}" + ], + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "i": "d64c1f90-53a1-442e-9f73-f3d17435b749", + "x": 0, + "y": 0, + "w": 1, + "h": 4, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Properties", + "description": "", + "timeframe": "none", + "dataStream": { + "id": "datastream-properties", + "name": "properties", + "metadata": [ + { + "name": "expiryStatus", + "displayName": "Expiry Status", + "computed": true, + "valueExpression": "{{{{raw}}}}{{ (v => { if (!v) return 'success'; const days = (new Date(v) - Date.now()) / 864e5; return days < 1 ? 'error' : days < 5 ? 'warning' : 'success'; })([].concat($['expires_at'])[0]) }}{{{{/raw}}}}", + "shape": [ + "state", + { + "map": { + "error": [ + "error" + ], + "warning": [ + "warning" + ], + "success": [ + "success" + ] + } + } + ] + } + ] + }, + "scope": { + "scope": "{{scopes.[API Keys]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[API Key]}}" + }, + "variables": [ + "{{variables.[API Key]}}" + ], + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": true, + "columnOrder": [ + "name", + "workspace_id", + "disabled", + "created_at", + "expires_at", + "expiryStatus" + ], + "hiddenColumns": [ + "id", + "links", + "type", + "label", + "sourceId", + "sourceType", + "sourceName", + "rawId" + ] + } + } + } + } + }, + { + "i": "a69c68dd-2968-4ac8-acc6-a224d85d7c37", + "x": 1, + "y": 0, + "w": 3, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend Over Time", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "filterObjects": { + "scope": "{{scopes.[API Keys]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[API Key]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[API Key]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Spend ($)", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "19cc22f9-e6fd-4c23-bd89-7e925a85849a", + "x": 1, + "y": 2, + "w": 3, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Requests Over Time", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "request_count", + "filterObjects": { + "scope": "{{scopes.[API Keys]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[API Key]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[API Key]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Requests", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "5c1cfeba-46df-41a0-aebc-9fcfb2a35a81", + "x": 0, + "y": 4, + "w": 4, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend By Model", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "groupBy": "model", + "filterObjects": { + "scope": "{{scopes.[API Keys]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[API Key]}}" + } + }, + "group": { + "by": [ + [ + "group", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "value" + ] + } + ] + }, + "sort": { + "by": [ + [ + "value_sum", + "desc" + ] + ], + "top": 10 + }, + "metadata": [ + { + "name": "value_sum", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[API Key]}}" + ], + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "group_uniqueValues", + "yAxisData": [ + "value_sum" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Spend ($)", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + } + ] + } +} diff --git a/plugins/OpenRouter/v1/defaultContent/manifest.json b/plugins/OpenRouter/v1/defaultContent/manifest.json new file mode 100644 index 00000000..3116f412 --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/manifest.json @@ -0,0 +1,24 @@ +{ + "items": [ + { + "name": "overview", + "type": "dashboard" + }, + { + "name": "modelComparison", + "type": "dashboard" + }, + { + "name": "model", + "type": "dashboard" + }, + { + "name": "workspace", + "type": "dashboard" + }, + { + "name": "apiKey", + "type": "dashboard" + } + ] +} diff --git a/plugins/OpenRouter/v1/defaultContent/model.dash.json b/plugins/OpenRouter/v1/defaultContent/model.dash.json new file mode 100644 index 00000000..505de92b --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/model.dash.json @@ -0,0 +1,593 @@ +{ + "name": "Model", + "schemaVersion": "1.5", + "timeframe": "last30days", + "variables": [ + "{{variables.[Model]}}" + ], + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "i": "f84087cc-45c8-4d32-9422-8e1b76bd7d37", + "x": 0, + "y": 0, + "w": 1, + "h": 4, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Properties", + "description": "", + "timeframe": "none", + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": true, + "columnOrder": [ + "name", + "author", + "canonical_slug", + "context_length" + ], + "hiddenColumns": [ + "id", + "links", + "type", + "label", + "sourceId", + "sourceType", + "sourceName", + "rawId" + ] + } + } + } + } + }, + { + "i": "aadaac03-632a-4219-8d5c-3ddb8696d938", + "x": 1, + "y": 0, + "w": 1, + "h": 4, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Pricing And Capabilities", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "models": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + } + } + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": true, + "columnOrder": [ + "inputPrice", + "outputPrice", + "cacheReadPrice", + "context_length", + "top_provider.max_completion_tokens", + "inputModalities", + "outputModalities", + "benchmarks.artificial_analysis.intelligence_index", + "benchmarks.artificial_analysis.coding_index", + "benchmarks.artificial_analysis.agentic_index", + "knowledge_cutoff", + "created", + "expiration_date" + ], + "hiddenColumns": [ + "id", + "name", + "author", + "canonical_slug", + "pricing.prompt", + "pricing.completion", + "pricing.input_cache_read", + "architecture.input_modalities", + "architecture.output_modalities" + ] + } + } + } + } + }, + { + "i": "5ddeb1c0-9941-4264-8695-b0d2e3a235ab", + "x": 2, + "y": 0, + "w": 2, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Spend", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "filterObjects": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Spend ($)", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "be276411-9f7c-4567-9d57-3222ced1f027", + "x": 2, + "y": 2, + "w": 2, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Requests", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "request_count", + "filterObjects": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Requests", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "21bb9221-3105-46cb-a988-b677dafef3e9", + "x": 0, + "y": 4, + "w": 4, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Providers", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[modelProviders]}}", + "name": "modelProviders", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "uptime_last_1d", + "desc" + ] + ] + } + }, + "scope": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "provider_name", + "tag", + "quantization", + "context_length", + "inputPrice", + "outputPrice", + "uptime_last_30m", + "uptime_last_1d", + "latency_last_30m.p50", + "latency_last_30m.p90", + "throughput_last_30m.p50" + ], + "hiddenColumns": [ + "model_name", + "pricing.prompt", + "pricing.completion" + ] + } + } + } + } + }, + { + "i": "c4ac55e8-5333-45eb-ace6-c5e7aad15b6f", + "x": 0, + "y": 7, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Latency By Provider", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[modelProviders]}}", + "name": "modelProviders", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "latency_last_30m.p50", + "asc" + ] + ] + } + }, + "scope": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "tag", + "yAxisData": [ + "latency_last_30m.p50", + "latency_last_30m.p90" + ], + "xAxisGroup": "none", + "xAxisLabel": "Provider", + "yAxisLabel": "Latency (ms)", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": true, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "e81a0ea0-b5fd-4021-8737-2c50997ab7a4", + "x": 2, + "y": 7, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Throughput By Provider", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[modelProviders]}}", + "name": "modelProviders", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "throughput_last_30m.p50", + "desc" + ] + ] + } + }, + "scope": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "tag", + "yAxisData": [ + "throughput_last_30m.p50" + ], + "xAxisGroup": "none", + "xAxisLabel": "Provider", + "yAxisLabel": "Tokens/s", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "041f6c43-1bf6-4e38-80bb-ca48e03ab8f0", + "x": 0, + "y": 10, + "w": 4, + "h": 7, + "config": { + "_type": "tile/data-stream", + "title": "Benchmarks Vs Price", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "models": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + } + } + }, + "variables": [ + "{{variables.[Model]}}" + ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "radar", + "encoding": { + "axes": [ + { + "field": "benchmarks.artificial_analysis.intelligence_index", + "type": "quantitative", + "title": "Intelligence Index" + }, + { + "field": "benchmarks.artificial_analysis.coding_index", + "type": "quantitative", + "title": "Coding Index" + }, + { + "field": "benchmarks.artificial_analysis.agentic_index", + "type": "quantitative", + "title": "Agentic Index" + }, + { + "field": "inputPrice", + "type": "quantitative", + "title": "Input Price ($/1M Tokens)" + }, + { + "field": "outputPrice", + "type": "quantitative", + "title": "Output Price ($/1M Tokens)" + }, + { + "field": "cacheReadPrice", + "type": "quantitative", + "title": "Cache Read Price ($/1M Tokens)" + } + ] + }, + "style": { + "legendPosition": "none", + "radarShape": "polygon" + } + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": true, + "columnOrder": [ + "inputPrice", + "outputPrice", + "cacheReadPrice", + "context_length", + "top_provider.max_completion_tokens", + "inputModalities", + "outputModalities", + "benchmarks.artificial_analysis.intelligence_index", + "benchmarks.artificial_analysis.coding_index", + "benchmarks.artificial_analysis.agentic_index", + "knowledge_cutoff", + "created", + "expiration_date" + ], + "hiddenColumns": [ + "id", + "name", + "author", + "canonical_slug", + "pricing.prompt", + "pricing.completion", + "pricing.input_cache_read", + "architecture.input_modalities", + "architecture.output_modalities" + ] + } + } + } + } + } + ] + } +} diff --git a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json new file mode 100644 index 00000000..cde0ad2c --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json @@ -0,0 +1,631 @@ +{ + "name": "Model Comparison", + "schemaVersion": "1.5", + "timeframe": "last30days", + "dashboard": { + "_type": "layout/grid", + "columns": 6, + "version": 1, + "contents": [ + { + "i": "0c29cf9b-46f4-478d-98c4-25a07523750b", + "x": 0, + "y": 0, + "w": 6, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Models", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "benchmarks.artificial_analysis.intelligence_index", + "desc" + ] + ] + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "name", + "author", + "inputPrice", + "outputPrice", + "context_length", + "benchmarks.artificial_analysis.intelligence_index", + "benchmarks.artificial_analysis.coding_index", + "benchmarks.artificial_analysis.agentic_index" + ], + "hiddenColumns": [ + "id", + "canonical_slug", + "cacheReadPrice", + "top_provider.max_completion_tokens", + "inputModalities", + "outputModalities", + "knowledge_cutoff", + "created", + "expiration_date", + "pricing.prompt", + "pricing.completion", + "pricing.input_cache_read", + "architecture.input_modalities", + "architecture.output_modalities" + ] + } + } + } + } + }, + { + "i": "3eec66c4-de77-40ee-b403-37620ac6ec6b", + "x": 0, + "y": 4, + "w": 2, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "Cost Per Task For Top GPQA Models", + "description": "By Accuracy", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "source": "openrouter" + }, + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "benchmark", + "operation": "equals", + "value": "gpqa_diamond" + }, + { + "column": "metric", + "operation": "equals", + "value": "accuracy" + } + ] + }, + "sort": { + "by": [ + [ + "score", + "desc" + ] + ], + "top": 15 + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "modelName", + "yAxisData": [ + "avgCostPerTask" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Avg Cost Per Task (USD)", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "df0c3dda-bfa5-48a0-babf-ed96590b4be2", + "x": 2, + "y": 4, + "w": 2, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "Top Models For Website Design (Elo)", + "description": "By Elo Rating (Design Arena)", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "source": "design-arena" + }, + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "benchmark", + "operation": "equals", + "value": "website" + }, + { + "column": "metric", + "operation": "equals", + "value": "elo" + } + ] + }, + "sort": { + "by": [ + [ + "score", + "desc" + ] + ], + "top": 15 + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "modelName", + "yAxisData": [ + "score" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Elo", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "3609a3d6-a8ff-470d-8333-20744922db53", + "x": 4, + "y": 4, + "w": 2, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "Top Models By Intelligence Index", + "description": "By Index Score (Artificial Analysis)", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "source": "artificial-analysis" + }, + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "benchmark", + "operation": "equals", + "value": "Intelligence Index" + }, + { + "column": "metric", + "operation": "equals", + "value": "index" + } + ] + }, + "sort": { + "by": [ + [ + "score", + "desc" + ] + ], + "top": 15 + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "modelName", + "yAxisData": [ + "score" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Intelligence Index", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "bdbfd6a7-4bbe-4eab-91b5-6908a10344d7", + "x": 0, + "y": 7, + "w": 3, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "Most Used Models On OpenRouter", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[rankings]}}", + "name": "rankings", + "pluginConfigId": "{{configId}}", + "group": { + "by": [ + [ + "model_permaslug", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "total_tokens" + ] + } + ] + }, + "sort": { + "by": [ + [ + "total_tokens_sum", + "desc" + ] + ], + "top": 10 + }, + "metadata": [ + { + "name": "total_tokens_sum", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "model_permaslug_uniqueValues", + "yAxisData": [ + "total_tokens_sum" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Tokens", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "22bac160-9e66-4597-b230-270811258056", + "x": 3, + "y": 7, + "w": 3, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "OpenRouter Token Share By Author", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[rankings]}}", + "name": "rankings", + "pluginConfigId": "{{configId}}", + "group": { + "by": [ + [ + "author", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "total_tokens" + ] + } + ] + }, + "sort": { + "by": [ + [ + "total_tokens_sum", + "desc" + ] + ] + }, + "metadata": [ + { + "name": "total_tokens_sum", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "total_tokens_sum", + "labelColumn": "author_uniqueValues", + "hideCenterValue": false, + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "i": "56d4d9f8-9cc4-4852-9306-2997d9e87000", + "x": 0, + "y": 10, + "w": 6, + "h": 8, + "config": { + "_type": "tile/data-stream", + "title": "Top Agentic Models Compared", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}", + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "benchmarks.artificial_analysis.agentic_index", + "operation": "greaterthan", + "value": "0" + }, + { + "column": "benchmarks.artificial_analysis.intelligence_index", + "operation": "greaterthan", + "value": "0" + }, + { + "column": "benchmarks.artificial_analysis.coding_index", + "operation": "greaterthan", + "value": "0" + }, + { + "column": "name", + "operation": "notcontains", + "value": "(batch)" + }, + { + "column": "name", + "operation": "notcontains", + "value": "(free)" + } + ] + }, + "sort": { + "by": [ + [ + "benchmarks.artificial_analysis.agentic_index", + "desc" + ], + [ + "benchmarks.artificial_analysis.intelligence_index", + "desc" + ], + [ + "benchmarks.artificial_analysis.coding_index", + "desc" + ] + ], + "top": 10 + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "radar", + "encoding": { + "series": { + "field": "name", + "type": "nominal", + "title": "Model" + }, + "axes": [ + { + "field": "benchmarks.artificial_analysis.intelligence_index", + "type": "quantitative", + "title": "Intelligence Index" + }, + { + "field": "benchmarks.artificial_analysis.coding_index", + "type": "quantitative", + "title": "Coding Index" + }, + { + "field": "benchmarks.artificial_analysis.agentic_index", + "type": "quantitative", + "title": "Agentic Index" + }, + { + "field": "inputPrice", + "type": "quantitative", + "title": "Input Price ($/1M Tokens)" + }, + { + "field": "outputPrice", + "type": "quantitative", + "title": "Output Price ($/1M Tokens)" + }, + { + "field": "cacheReadPrice", + "type": "quantitative", + "title": "Cache Read Price ($/1M Tokens)" + } + ] + }, + "style": { + "legendPosition": "right", + "radarShape": "polygon", + "radarFill": false, + "radarRadius": 80 + } + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "columnOrder": [ + "name", + "author", + "benchmarks.artificial_analysis.intelligence_index", + "benchmarks.artificial_analysis.coding_index", + "benchmarks.artificial_analysis.agentic_index", + "inputPrice", + "outputPrice", + "cacheReadPrice" + ], + "hiddenColumns": [ + "id", + "canonical_slug", + "cacheReadPrice", + "top_provider.max_completion_tokens", + "inputModalities", + "outputModalities", + "knowledge_cutoff", + "created", + "expiration_date", + "pricing.prompt", + "pricing.completion", + "pricing.input_cache_read", + "architecture.input_modalities", + "architecture.output_modalities" + ] + } + } + } + } + }, + { + "i": "679f0166-77c4-47c1-9ae5-7e4c1721f718", + "x": 0, + "y": 18, + "w": 6, + "h": 1, + "config": { + "_type": "tile/text", + "title": "", + "description": "", + "visualisation": { + "config": { + "content": "Benchmark data from Artificial Analysis, Design Arena, and OpenRouter, via OpenRouter \nSource: [OpenRouter](https://openrouter.ai/rankings), licensed CC BY 4.0", + "fontSize": 12, + "align": "left", + "autoSize": true + } + } + } + } + ] + } +} diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json new file mode 100644 index 00000000..833f63a2 --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -0,0 +1,574 @@ +{ + "name": "Overview", + "schemaVersion": "1.5", + "timeframe": "last30days", + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "i": "bb7e5c59-ac7d-4c28-9966-a72354c327b2", + "x": 0, + "y": 0, + "w": 1, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[credits]}}", + "name": "credits", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": "remaining", + "comparisonColumn": "none", + "label": "Credits Remaining" + } + } + } + } + }, + { + "i": "5f747a12-0c82-45c6-932d-734b1c4ac88d", + "x": 1, + "y": 0, + "w": 1, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[credits]}}", + "name": "credits", + "pluginConfigId": "{{configId}}" + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": "total_usage", + "comparisonColumn": "none", + "label": "Credits Used" + } + } + } + } + }, + { + "i": "6143e7be-4b44-4ff1-804c-ada7021bd6d6", + "x": 2, + "y": 0, + "w": 1, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "", + "description": "", + "timeframe": "thisMonth", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage" + }, + "group": { + "by": [], + "aggregate": [ + { + "type": "sum", + "names": [ + "value" + ] + } + ] + }, + "metadata": [ + { + "name": "value_sum", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": "value_sum", + "comparisonColumn": "none", + "label": "Spend (This Month)" + } + } + } + } + }, + { + "i": "d77eec0c-ba8b-48a5-a65d-86fcac004f4d", + "x": 3, + "y": 0, + "w": 1, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "", + "description": "", + "timeframe": "thisMonth", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "request_count" + }, + "group": { + "by": [], + "aggregate": [ + { + "type": "sum", + "names": [ + "value" + ] + } + ] + }, + "metadata": [ + { + "name": "value_sum", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-scalar", + "config": { + "data-stream-scalar": { + "value": "value_sum", + "comparisonColumn": "none", + "label": "Requests (This Month)" + } + } + } + } + }, + { + "i": "63530d85-be7f-4e7e-bd5f-645da0d4bb6e", + "x": 0, + "y": 2, + "w": 2, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend Over Time", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage" + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Spend ($)", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "86f14850-5e5d-40dc-83ed-e0cf5b90684a", + "x": 2, + "y": 2, + "w": 2, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Requests Over Time", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "request_count" + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Requests", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "55c667d9-63af-4556-962d-87a6325df855", + "x": 0, + "y": 4, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend By Model", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "groupBy": "model" + }, + "group": { + "by": [ + [ + "group", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "value" + ] + } + ] + }, + "sort": { + "by": [ + [ + "value_sum", + "desc" + ] + ], + "top": 10 + }, + "metadata": [ + { + "name": "value_sum", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "group_uniqueValues", + "yAxisData": [ + "value_sum" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Spend ($)", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "7c94001a-4946-45e0-8d08-08f2b89153ef", + "x": 2, + "y": 4, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend By Workspace", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "groupBy": "workspace" + }, + "group": { + "by": [ + [ + "group", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "value" + ] + } + ] + }, + "sort": { + "by": [ + [ + "value_sum", + "desc" + ] + ], + "top": 10 + }, + "metadata": [ + { + "name": "value_sum", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "value_sum", + "labelColumn": "group_uniqueValues", + "hideCenterValue": false, + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "i": "d596864a-98e3-466c-ba4d-cd59ee6f7f01", + "x": 0, + "y": 7, + "w": 4, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Top API Keys By Spend (This Month)", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[apiKeys]}}", + "name": "apiKeys", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "usage_monthly", + "desc" + ] + ], + "top": 10 + } + }, + "scope": { + "scope": "{{scopes.[Workspaces]}}", + "workspace": "{{workspaceId}}" + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "name", + "usage_monthly", + "usage_weekly", + "usage_daily", + "usage", + "limit", + "limit_remaining", + "limit_reset", + "disabled" + ], + "hiddenColumns": [ + "hash", + "workspace_id", + "created_at", + "expires_at" + ] + } + } + } + } + } + ] + } +} diff --git a/plugins/OpenRouter/v1/defaultContent/scopes.json b/plugins/OpenRouter/v1/defaultContent/scopes.json new file mode 100644 index 00000000..59998e4e --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/scopes.json @@ -0,0 +1,53 @@ +[ + { + "name": "Models", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Model" + ] + } + }, + "variable": { + "name": "Model", + "type": "object", + "default": "none", + "allowMultipleSelection": false + } + }, + { + "name": "Workspaces", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Workspace" + ] + } + }, + "variable": { + "name": "Workspace", + "type": "object", + "default": "none", + "allowMultipleSelection": false + } + }, + { + "name": "API Keys", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "API Key" + ] + } + }, + "variable": { + "name": "API Key", + "type": "object", + "default": "none", + "allowMultipleSelection": false + } + } +] diff --git a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json new file mode 100644 index 00000000..a21ba7fc --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json @@ -0,0 +1,468 @@ +{ + "name": "Workspace", + "schemaVersion": "1.5", + "timeframe": "last30days", + "variables": [ + "{{variables.[Workspace]}}" + ], + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "i": "ff915e58-be26-44f9-ac25-6ad57c347e6e", + "x": 0, + "y": 0, + "w": 1, + "h": 4, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Properties", + "description": "", + "timeframe": "none", + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Workspaces]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Workspace]}}" + }, + "variables": [ + "{{variables.[Workspace]}}" + ], + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": true, + "columnOrder": [ + "name", + "slug" + ], + "hiddenColumns": [ + "id", + "links", + "type", + "label", + "sourceId", + "sourceType", + "sourceName", + "rawId" + ] + } + } + } + } + }, + { + "i": "4b027f1f-ba96-4972-af3f-5b397fb9496f", + "x": 1, + "y": 0, + "w": 3, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend Over Time", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "filterObjects": { + "scope": "{{scopes.[Workspaces]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Workspace]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Workspace]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Spend ($)", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "2ef51ad6-de4b-4059-a907-ea2f57b88309", + "x": 1, + "y": 2, + "w": 3, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Requests Over Time", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "request_count", + "filterObjects": { + "scope": "{{scopes.[Workspaces]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Workspace]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Workspace]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Requests", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "98c1dbfc-ae37-4636-8ea3-8ab5ed8b56e3", + "x": 0, + "y": 4, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend By Model", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "groupBy": "model", + "filterObjects": { + "scope": "{{scopes.[Workspaces]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Workspace]}}" + } + }, + "group": { + "by": [ + [ + "group", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "value" + ] + } + ] + }, + "sort": { + "by": [ + [ + "value_sum", + "desc" + ] + ], + "top": 10 + }, + "metadata": [ + { + "name": "value_sum", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Workspace]}}" + ], + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "group_uniqueValues", + "yAxisData": [ + "value_sum" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Spend ($)", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "4d208d06-5051-4bfe-8834-ddef5b530eac", + "x": 2, + "y": 4, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend By API Key", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "groupBy": "api_key_id", + "filterObjects": { + "scope": "{{scopes.[Workspaces]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Workspace]}}" + } + }, + "group": { + "by": [ + [ + "group", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "value" + ] + } + ] + }, + "sort": { + "by": [ + [ + "value_sum", + "desc" + ] + ], + "top": 10 + }, + "metadata": [ + { + "name": "value_sum", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Workspace]}}" + ], + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "value_sum", + "labelColumn": "group_uniqueValues", + "hideCenterValue": false, + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "i": "34c93f18-f42c-442f-bd08-2bfba9a897d0", + "x": 0, + "y": 7, + "w": 4, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "API Keys", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[apiKeys]}}", + "name": "apiKeys", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "usage_monthly", + "desc" + ] + ] + } + }, + "scope": { + "scope": "{{scopes.[Workspaces]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Workspace]}}" + }, + "variables": [ + "{{variables.[Workspace]}}" + ], + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "name", + "usage_monthly", + "usage_weekly", + "usage_daily", + "usage", + "limit", + "limit_remaining", + "limit_reset", + "disabled" + ], + "hiddenColumns": [ + "hash", + "workspace_id", + "created_at", + "expires_at" + ] + } + } + } + } + } + ] + } +} diff --git a/plugins/OpenRouter/v1/docs/README.md b/plugins/OpenRouter/v1/docs/README.md new file mode 100644 index 00000000..e78a2d47 --- /dev/null +++ b/plugins/OpenRouter/v1/docs/README.md @@ -0,0 +1,73 @@ +Monitor your [OpenRouter](https://openrouter.ai) spend, usage, and API keys in SquaredUp via the [OpenRouter API](https://openrouter.ai/docs/api_reference/overview). Compare models by price, benchmarks, and provider performance. + +## Setup + +You will need an OpenRouter **management key** for full access. A standard API key also works, but only for model, benchmark, ranking, and credit data. + +| Key type | What works | +| -------- | ---------- | +| **Management key** | Everything: models, benchmarks, rankings, credits, workspaces, API keys, and usage. | +| **Standard API key** | Models, benchmarks, rankings, and credits. Workspaces, API keys, and usage are unavailable. | + +Management keys can read and manage account, workspace, API key, and usage data, but cannot be used to call models. This plugin only reads data; it never creates, modifies, or deletes anything in OpenRouter. + +1. Sign in to [OpenRouter](https://openrouter.ai). Organization accounts require the **Admin** role to create a management key. +2. Go to [**Settings → Management Keys**](https://openrouter.ai/settings/management-keys) and create a key. See the [management key documentation](https://openrouter.ai/docs/guides/overview/auth/management-api-keys) for details. +3. To use a standard key instead, create one under [**Settings → API Keys**](https://openrouter.ai/settings/keys). A **$0** credit limit works, so the key can read data but never spend credit. +4. Copy the key and paste it into the **API key** field. + +## Configuration fields + +| Field | What it is | Where to find it | Required | +| ----------- | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | -------- | +| **API key** | A management key (recommended) or standard API key, sent as a Bearer token. | OpenRouter → **Settings → Management Keys** (or **Settings → API Keys**). | Yes | + +On save, the plugin checks the key in two steps. An invalid, disabled, or expired key fails setup. A valid standard API key passes with a warning that workspaces, API keys, and usage need a management key. + +## What this plugin monitors + +- **Spend and usage** — spend, requests, and tokens over time, broken down by model, provider, workspace, API key, or app. +- **API keys** — spend today, this week, and this month for each key, against its credit limit, plus an expiry warning. +- **Account credits** — credits purchased, used, and remaining. +- **Model comparison** — pricing, context length, modalities, and Artificial Analysis benchmark scores for every model on OpenRouter. +- **Provider performance** — price, uptime, latency, and throughput for each provider serving a model. +- **Benchmarks and rankings** — scores from Artificial Analysis, Design Arena, and OpenRouter's own evals, and OpenRouter's public leaderboard of the most used models. + +The out-of-the-box dashboards include an account **Overview**, a **Model Comparison** dashboard (which works with a standard API key), and a perspective for each **Model**, **Workspace**, and **API Key** (spend tiles need a management key). + +## Data streams + +- **Usage** — spend, requests, tokens, or cache hit rate over time, optionally grouped by model, provider, workspace, API key, or app, and optionally filtered to selected objects. Account-wide. Management key only. +- **API Keys** — API keys in a workspace with spend and limits. Per workspace. Management key only. +- **Workspaces** — workspaces in the account. Account-wide. Management key only. +- **Credits** — credits purchased, used, and remaining. Account-wide. +- **Models** — every model on OpenRouter with pricing ($ per 1M tokens), context length, modalities, and Artificial Analysis index scores, optionally filtered to selected models. +- **Model Providers** — providers serving a model, with price, uptime, latency, and throughput over the last 30 minutes. Per model. +- **Benchmarks** — model benchmark scores from Artificial Analysis, Design Arena, and OpenRouter evals, one row per model and benchmark. +- **Model Rankings** — the most used models across all OpenRouter traffic, by tokens processed per day. + +## What gets indexed + +| Object type | API source | Represents | +| ------------- | ------------------------------- | ------------------------------------------------------------------------ | +| **Model** | `GET /models` | A model available through OpenRouter, such as `anthropic/claude-sonnet-4.5`. | +| **Workspace** | `GET /workspaces` | A workspace in the account. Management key only. | +| **API Key** | `GET /keys` (per workspace) | An API key in a workspace, including its expiry date. Management key only. | + +**Relationships:** each API Key records the ID of its Workspace. + +With a standard API key, only models are imported; the workspace and API key import steps are skipped with a warning. + +## Known limitations + +- **Benchmarks and rankings refresh at most once a day** — OpenRouter limits these endpoints to 500 requests a day per account, so the plugin caches them for 24 hours. +- **Rankings licence** — rankings data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). The **Model Comparison** dashboard credits it already; keep "Source: OpenRouter (openrouter.ai/rankings)" on any dashboard you build from it. +- **Not every model has benchmark scores** — Artificial Analysis scores cover fewer than half of models; the rest show blank. +- **Usage history** can be queried up to a year back. Grouping usage by **Provider** is limited to 31 days; longer timeframes return an error. +- **Usage timeframes start at 12 hours** — data is bucketed by hour, day, week, or month to suit the range; periods with no activity are left out rather than shown as zero. +- **Large usage queries may be truncated** — the stream warns when this happens; narrow the timeframe or filter to specific objects. +- **Model rankings start at 7 days** — the data is daily. +- **Provider performance is a live snapshot** — it covers the last 30 minutes, with no history beyond what SquaredUp records. +- **Filtering usage needs imported objects** — pick workspaces and API keys in **Filter to objects (optional)**; grouped results show their names. +- **Organization members aren't imported.** +- **Read-only** — the plugin never creates, modifies, or deletes anything in OpenRouter. diff --git a/plugins/OpenRouter/v1/icon.svg b/plugins/OpenRouter/v1/icon.svg new file mode 100644 index 00000000..c3bc9f7d --- /dev/null +++ b/plugins/OpenRouter/v1/icon.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/plugins/OpenRouter/v1/indexDefinitions/default.json b/plugins/OpenRouter/v1/indexDefinitions/default.json new file mode 100644 index 00000000..9b180b54 --- /dev/null +++ b/plugins/OpenRouter/v1/indexDefinitions/default.json @@ -0,0 +1,55 @@ +{ + "steps": [ + { + "name": "models", + "dataStream": { + "name": "models" + }, + "timeframe": "none", + "objectMapping": { + "id": "id", + "name": "name", + "type": { + "value": "Model" + }, + "properties": ["author", "canonical_slug", "context_length"] + } + }, + { + "name": "workspaces", + "dataStream": { + "name": "workspaces" + }, + "timeframe": "none", + "objectMapping": { + "id": "id", + "name": "name", + "type": { + "value": "Workspace" + }, + "properties": ["slug"] + }, + "optional": true + }, + { + "name": "apiKeys", + "dataStream": { + "name": "apiKeys" + }, + "scope": { + "query": "g.V().has(\"sourceType\", \"Workspace\")" + }, + "timeframe": "none", + "objectMapping": { + "id": "hash", + "name": "name", + "type": { + "value": "API Key" + }, + "properties": ["workspace_id", "disabled", "created_at", "expires_at"] + }, + "optional": true, + "dependsOn": ["workspaces"] + } + ] +} diff --git a/plugins/OpenRouter/v1/metadata.json b/plugins/OpenRouter/v1/metadata.json new file mode 100644 index 00000000..b5f37d33 --- /dev/null +++ b/plugins/OpenRouter/v1/metadata.json @@ -0,0 +1,54 @@ +{ + "name": "openrouter", + "displayName": "OpenRouter", + "version": "1.0.0", + "author": { + "name": "@AnEvilPenguin", + "type": "community" + }, + "description": "Track OpenRouter spend and usage by workspace, API key, and model, and compare models by price, benchmarks, and provider performance.", + "category": "Monitoring", + "type": "cloud", + "schemaVersion": "2.1", + "importNotSupported": false, + "restrictedToPlatforms": [], + "keywords": [ + "openrouter", + "ai", + "llm", + "models", + "benchmarks", + "cost", + "usage", + "tokens", + "billing" + ], + "objectTypes": ["Model", "Workspace", "API Key"], + "links": [ + { + "category": "documentation", + "url": "https://github.com/squaredup/plugins/blob/main/plugins/OpenRouter/v1/docs/README.md", + "label": "Help adding this plugin" + }, + { + "category": "source", + "url": "https://github.com/squaredup/plugins/tree/main/plugins/OpenRouter/v1", + "label": "Repository" + } + ], + "base": { + "plugin": "WebAPI", + "majorVersion": "1", + "config": { + "baseUrl": "https://openrouter.ai/api/v1/", + "authMode": "none", + "headers": [ + { + "key": "Authorization", + "value": "Bearer {{apiKey}}" + } + ], + "queryArgs": [] + } + } +} diff --git a/plugins/OpenRouter/v1/ui.json b/plugins/OpenRouter/v1/ui.json new file mode 100644 index 00000000..e5b57f96 --- /dev/null +++ b/plugins/OpenRouter/v1/ui.json @@ -0,0 +1,12 @@ +[ + { + "type": "password", + "name": "apiKey", + "label": "API key", + "placeholder": "sk-or-v1-...", + "validation": { + "required": true + }, + "help": "Use a [management key](https://openrouter.ai/settings/management-keys) for full access to workspace, API key, and usage data. A standard [API key](https://openrouter.ai/settings/keys) covers model, benchmark, ranking, and credit data only. This plugin only reads data; it does not create, modify, or delete anything in OpenRouter" + } +] From 75c0a4f2b6f16e518189327d2465b02d87622afd Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Fri, 2 Oct 2026 19:02:16 +0100 Subject: [PATCH 02/16] Add OpenRouter performance metrics, providers, and task classifications Squashed from: - Add latency, throughput, and token metrics to OpenRouter usage - Import OpenRouter providers and add a Provider perspective - Add OpenRouter task classifications stream - Add status, 5m uptime, and more latency detail to OpenRouter model providers - Add smartViz tiles and top-N model rankings to OpenRouter dashboards - Make OpenRouter usage metric and group by grouped autocompletes Co-Authored-By: Claude Opus 5.5 --- plugins/OpenRouter/v1/custom_types.json | 7 + .../v1/dataStreams/modelProviders.json | 55 +- .../OpenRouter/v1/dataStreams/providers.json | 74 +++ .../OpenRouter/v1/dataStreams/rankings.json | 8 +- .../v1/dataStreams/scripts/providers.js | 3 + .../v1/dataStreams/scripts/rankings.js | 41 ++ .../scripts/taskClassifications.js | 20 + .../v1/dataStreams/scripts/usage.js | 7 +- .../v1/dataStreams/taskClassifications.json | 85 ++++ plugins/OpenRouter/v1/dataStreams/usage.json | 199 +++++--- .../v1/defaultContent/manifest.json | 4 + .../v1/defaultContent/model.dash.json | 318 +++++++++++- .../defaultContent/modelComparison.dash.json | 411 ++++++++++++++- .../v1/defaultContent/overview.dash.json | 291 ++++++++++- .../v1/defaultContent/provider.dash.json | 469 ++++++++++++++++++ .../OpenRouter/v1/defaultContent/scopes.json | 17 + plugins/OpenRouter/v1/docs/README.md | 32 +- .../v1/indexDefinitions/default.json | 15 + plugins/OpenRouter/v1/metadata.json | 2 +- 19 files changed, 1972 insertions(+), 86 deletions(-) create mode 100644 plugins/OpenRouter/v1/dataStreams/providers.json create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/providers.js create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/rankings.js create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/taskClassifications.js create mode 100644 plugins/OpenRouter/v1/dataStreams/taskClassifications.json create mode 100644 plugins/OpenRouter/v1/defaultContent/provider.dash.json diff --git a/plugins/OpenRouter/v1/custom_types.json b/plugins/OpenRouter/v1/custom_types.json index 72926835..71d74943 100644 --- a/plugins/OpenRouter/v1/custom_types.json +++ b/plugins/OpenRouter/v1/custom_types.json @@ -6,6 +6,13 @@ "singular": "Model", "plural": "Models" }, + { + "name": "Provider", + "sourceType": "Provider", + "icon": "server", + "singular": "Provider", + "plural": "Providers" + }, { "name": "Workspace", "sourceType": "Workspace", diff --git a/plugins/OpenRouter/v1/dataStreams/modelProviders.json b/plugins/OpenRouter/v1/dataStreams/modelProviders.json index ad0d00c8..08a4d91d 100644 --- a/plugins/OpenRouter/v1/dataStreams/modelProviders.json +++ b/plugins/OpenRouter/v1/dataStreams/modelProviders.json @@ -1,7 +1,7 @@ { "name": "modelProviders", "displayName": "Model Providers", - "description": "Providers serving a model, with price, uptime, latency, and throughput (last 30 minutes)", + "description": "Providers serving a model, with status, price, uptime, latency, and throughput (last 5 minutes to 1 day)", "tags": [ "Models", "Performance" @@ -38,6 +38,19 @@ "shape": "string", "role": "label" }, + { + "name": "status", + "visible": false, + "displayName": "Status Code", + "shape": "number" + }, + { + "name": "health", + "displayName": "Status", + "shape": "state", + "computed": true, + "valueExpression": "{{ (s => s == null || s === '' ? 'unknown' : (n => n === 0 ? 'success' : n >= -3 ? 'warning' : 'error')(Number(s)))($['status']) }}" + }, { "name": "tag", "displayName": "Tag", @@ -103,6 +116,36 @@ "computed": true, "valueExpression": "{{ (v => v == null || v === '' || Number(v) < 0 ? null : Number(v) * 1e6)($['pricing.completion']) }}" }, + { + "name": "pricing.input_cache_read", + "visible": false, + "displayName": "Cache Read Price (per Token)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 8 + } + ] + }, + { + "name": "cacheReadPrice", + "displayName": "Cache Read Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ], + "computed": true, + "valueExpression": "{{ (v => v == null || v === '' || Number(v) < 0 ? null : Number(v) * 1e6)($['pricing.input_cache_read']) }}" + }, + { + "name": "uptime_last_5m", + "displayName": "Uptime (5m)", + "shape": "percent" + }, { "name": "uptime_last_30m", "displayName": "Uptime (30m)", @@ -118,11 +161,21 @@ "displayName": "Latency p50", "shape": "milliseconds" }, + { + "name": "latency_last_30m.p75", + "displayName": "Latency p75", + "shape": "milliseconds" + }, { "name": "latency_last_30m.p90", "displayName": "Latency p90", "shape": "milliseconds" }, + { + "name": "latency_last_30m.p99", + "displayName": "Latency p99", + "shape": "milliseconds" + }, { "name": "throughput_last_30m.p50", "displayName": "Throughput p50 (Tokens/s)", diff --git a/plugins/OpenRouter/v1/dataStreams/providers.json b/plugins/OpenRouter/v1/dataStreams/providers.json new file mode 100644 index 00000000..06a1a350 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/providers.json @@ -0,0 +1,74 @@ +{ + "name": "providers", + "displayName": "Providers", + "description": "Providers serving models on OpenRouter, with headquarters, datacenter locations, and policy links", + "tags": [ + "Providers" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "providers", + "paging": { + "mode": "none" + }, + "getArgs": [], + "headers": [], + "postRequestScript": "providers.js" + }, + "matches": "none", + "metadata": [ + { + "name": "slug", + "displayName": "Provider ID", + "shape": "string" + }, + { + "name": "name", + "displayName": "Name", + "shape": "string", + "role": "label" + }, + { + "name": "headquarters", + "displayName": "Headquarters", + "shape": "string" + }, + { + "name": "datacenters", + "displayName": "Datacenters", + "shape": "string" + }, + { + "name": "status_page_url", + "displayName": "Status Page", + "shape": [ + "url", + { + "label": "Status page" + } + ] + }, + { + "name": "privacy_policy_url", + "displayName": "Privacy Policy", + "shape": [ + "url", + { + "label": "Privacy policy" + } + ] + }, + { + "name": "terms_of_service_url", + "displayName": "Terms Of Service", + "shape": [ + "url", + { + "label": "Terms of service" + } + ] + } + ], + "timeframes": false +} diff --git a/plugins/OpenRouter/v1/dataStreams/rankings.json b/plugins/OpenRouter/v1/dataStreams/rankings.json index e238fa0b..87bcee95 100644 --- a/plugins/OpenRouter/v1/dataStreams/rankings.json +++ b/plugins/OpenRouter/v1/dataStreams/rankings.json @@ -32,7 +32,7 @@ } ], "headers": [], - "pathToData": "data" + "postRequestScript": "rankings.js" }, "matches": "none", "ui": [ @@ -109,6 +109,12 @@ "label": "Academia" } ] + }, + { + "type": "number", + "name": "top", + "label": "Top models (optional)", + "help": "Keep the models with the most tokens across the timeframe and combine the rest into **Other**" } ], "metadata": [ diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/providers.js b/plugins/OpenRouter/v1/dataStreams/scripts/providers.js new file mode 100644 index 00000000..7fe8f5d6 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/providers.js @@ -0,0 +1,3 @@ +// Flattens the datacenters country-code array into a comma-separated string +// so it can be shown in tables and indexed as a Provider property. +result = (data?.data ?? []).map((p) => ({ ...p, datacenters: (p.datacenters ?? []).join(', ') })); diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js b/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js new file mode 100644 index 00000000..440123d9 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js @@ -0,0 +1,41 @@ +// Optional `top`: keep the N models with the most tokens across the window and +// merge the rest (including OpenRouter's own `other` row) into one "Other" row +// per date, so stacked charts stay readable. Without it, rows pass through. +const rows = data?.data ?? []; +const top = Number(context.config.top); + +if (!(top > 0)) { + result = rows; +} else { + const totals = {}; + + for (const r of rows) { + if (r.model_permaslug === 'other') continue; + totals[r.model_permaslug] = (totals[r.model_permaslug] ?? 0) + Number(r.total_tokens); + } + + const kept = new Set( + Object.entries(totals) + .sort(([, a], [, b]) => b - a) + .slice(0, top) + .map(([model]) => model), + ); + + const otherByDate = {}; + + result = []; + + for (const r of rows) { + if (kept.has(r.model_permaslug)) { + result.push({ ...r, total_tokens: Number(r.total_tokens) }); + } else { + otherByDate[r.date] = (otherByDate[r.date] ?? 0) + Number(r.total_tokens); + } + } + + for (const [date, total_tokens] of Object.entries(otherByDate)) { + result.push({ date, model_permaslug: 'Other', total_tokens }); + } + + result.sort((a, b) => a.date.localeCompare(b.date)); +} diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/taskClassifications.js b/plugins/OpenRouter/v1/dataStreams/scripts/taskClassifications.js new file mode 100644 index 00000000..9011fb73 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/taskClassifications.js @@ -0,0 +1,20 @@ +// One row per task classification, with its macro-category label and the +// task's most used model. Shares are fractions (0-1) of sampled traffic. +const body = data?.data ?? {}; +const categoryLabels = Object.fromEntries((body.macro_categories ?? []).map((c) => [c.key, c.label])); + +result = (body.classifications ?? []).map((c) => { + // models isn't ordered by request share, so pick the largest explicitly + const topModel = (c.models ?? []).reduce((top, m) => (!top || m.tag_usage_share > top.tag_usage_share ? m : top), null); + + return { + tag: c.tag, + task: c.display_name, + category: categoryLabels[c.macro_category] ?? c.macro_category, + usageShare: c.usage_share, + tokenShare: c.token_share, + topModel: topModel?.id ?? null, + topModelUsageShare: topModel?.tag_usage_share ?? null, + asOf: body.as_of, + }; +}); diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/usage.js b/plugins/OpenRouter/v1/dataStreams/scripts/usage.js index 6e797450..6bd542f3 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/usage.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/usage.js @@ -1,5 +1,6 @@ -// Normalises analytics rows: the time key is named after the granularity -// (date__day, date__hour, ...) and large counts arrive as strings. +// Normalises analytics rows: the time key is suffixed with the granularity +// (date__day, or created_at__day for raw-view fields like latency and provider) +// and large counts arrive as strings. const body = data?.data ?? {}; const metric = context.config.metric || 'total_usage'; const groupBy = context.config.groupBy; @@ -10,7 +11,7 @@ if (body.metadata?.truncated) { ); } -const dateKey = (row) => Object.keys(row).find((k) => k.startsWith('date__')); +const dateKey = (row) => Object.keys(row).find((k) => /__(minute|hour|day|week|month)$/.test(k)); result = (body.data ?? []) .map((row) => { diff --git a/plugins/OpenRouter/v1/dataStreams/taskClassifications.json b/plugins/OpenRouter/v1/dataStreams/taskClassifications.json new file mode 100644 index 00000000..2600cc56 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/taskClassifications.json @@ -0,0 +1,85 @@ +{ + "name": "taskClassifications", + "displayName": "Task Classifications", + "description": "Share of OpenRouter traffic by task type over the last 7 days, with each task's most used model (from OpenRouter's public rankings)", + "tags": [ + "Models", + "Usage" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "classifications/task", + "paging": { + "mode": "none" + }, + "getArgs": [], + "headers": [], + "postRequestScript": "taskClassifications.js" + }, + "matches": "none", + "metadata": [ + { + "name": "tag", + "displayName": "Tag", + "shape": "string", + "visible": false + }, + { + "name": "task", + "displayName": "Task", + "shape": "string", + "role": "label" + }, + { + "name": "category", + "displayName": "Category", + "shape": "string" + }, + { + "name": "usageShare", + "displayName": "Request Share", + "shape": [ + "percent", + { + "asZeroToOne": true + } + ], + "role": "value" + }, + { + "name": "tokenShare", + "displayName": "Token Share", + "shape": [ + "percent", + { + "asZeroToOne": true + } + ] + }, + { + "name": "topModel", + "displayName": "Top Model", + "shape": "string" + }, + { + "name": "topModelUsageShare", + "displayName": "Top Model Share Of Task Requests", + "shape": [ + "percent", + { + "asZeroToOne": true + } + ] + }, + { + "name": "asOf", + "displayName": "As Of", + "shape": "date" + } + ], + "options": { + "minimumRequestIntervalSeconds": 86400 + }, + "timeframes": false +} diff --git a/plugins/OpenRouter/v1/dataStreams/usage.json b/plugins/OpenRouter/v1/dataStreams/usage.json index 460cc43c..652e0e82 100644 --- a/plugins/OpenRouter/v1/dataStreams/usage.json +++ b/plugins/OpenRouter/v1/dataStreams/usage.json @@ -1,7 +1,7 @@ { "name": "usage", "displayName": "Usage", - "description": "Spend, requests, and tokens over time, optionally grouped by model, provider, workspace, API key, or app (requires a management key)", + "description": "Spend, requests, tokens, latency, and throughput over time, optionally grouped by model, provider, workspace, API key, or app (requires a management key)", "tags": [ "Usage", "Cost" @@ -15,76 +15,162 @@ }, "getArgs": [], "headers": [], - "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; const days = (new Date(timeframe.end) - new Date(timeframe.start)) / 864e5; const granularity = days <= 2 ? 'hour' : days <= 31 ? 'day' : days <= 92 ? 'week' : 'month'; const selected = typeof filterObjects !== 'undefined' && Array.isArray(filterObjects) ? filterObjects : []; const one = (v) => (Array.isArray(v) ? v[0] : v); const idsOf = (type, ...props) => [...new Set(selected.filter((o) => one(o.sourceType) === type).flatMap((o) => props.map((p) => one(o[p]))).filter(Boolean))]; const rules = [ ['model', idsOf('Model', 'rawId', 'canonical_slug')], ['workspace', idsOf('Workspace', 'rawId')], ['api_key_id', idsOf('API Key', 'rawId')] ]; const filters = rules.filter(([, values]) => values.length).map(([field, value]) => ({ field, operator: 'in', value })); return JSON.stringify({ metrics: [typeof metric !== 'undefined' && metric ? metric : 'total_usage'], dimensions: typeof groupBy !== 'undefined' && groupBy ? [groupBy] : [], filters, granularity, time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", + "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; const days = (new Date(timeframe.end) - new Date(timeframe.start)) / 864e5; const granularity = days <= 2 ? 'hour' : days <= 31 ? 'day' : days <= 92 ? 'week' : 'month'; const selected = typeof filterObjects !== 'undefined' && Array.isArray(filterObjects) ? filterObjects : []; const one = (v) => (Array.isArray(v) ? v[0] : v); const idsOf = (type, ...props) => [...new Set(selected.filter((o) => one(o.sourceType) === type).flatMap((o) => props.map((p) => one(o[p]))).filter(Boolean))]; const rules = [ ['model', idsOf('Model', 'rawId', 'canonical_slug')], ['provider', idsOf('Provider', 'name')], ['workspace', idsOf('Workspace', 'rawId')], ['api_key_id', idsOf('API Key', 'rawId')] ]; const filters = rules.filter(([, values]) => values.length).map(([field, value]) => ({ field, operator: 'in', value })); return JSON.stringify({ metrics: [typeof metric !== 'undefined' && metric ? metric : 'total_usage'], dimensions: typeof groupBy !== 'undefined' && groupBy ? [groupBy] : [], filters, granularity, time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", "postRequestScript": "usage.js" }, "matches": "none", "ui": [ { - "type": "choiceChips", + "type": "autocomplete", "name": "metric", "label": "Metric", "defaultValue": "total_usage", + "help": "Select a metric, or enter any metric from [OpenRouter's analytics metrics](https://openrouter.ai/docs/api/api-reference/analytics/get-available-analytics-metrics-and-dimensions). Performance metrics are limited to 31 days.", "validation": { "required": true }, - "options": [ - { - "value": "total_usage", - "label": "Spend (USD)" - }, - { - "value": "request_count", - "label": "Requests" - }, - { - "value": "tokens_total", - "label": "Total tokens" - }, - { - "value": "tokens_prompt", - "label": "Prompt tokens" - }, - { - "value": "tokens_completion", - "label": "Completion tokens" - }, - { - "value": "reasoning_tokens", - "label": "Reasoning tokens" - }, - { - "value": "cache_hit_rate", - "label": "Cache hit rate" - } - ] + "allowCustomValues": true, + "isMulti": false, + "data": { + "source": "fixed", + "values": [ + { + "label": "Cost", + "options": [ + { + "value": "total_usage", + "label": "Spend (USD)" + }, + { + "value": "byok_usage", + "label": "BYOK spend (USD)" + } + ] + }, + { + "label": "Requests and tokens", + "options": [ + { + "value": "request_count", + "label": "Requests" + }, + { + "value": "tokens_total", + "label": "Total tokens" + }, + { + "value": "tokens_prompt", + "label": "Prompt tokens" + }, + { + "value": "tokens_completion", + "label": "Completion tokens" + }, + { + "value": "reasoning_tokens", + "label": "Reasoning tokens" + }, + { + "value": "cached_tokens", + "label": "Cached tokens" + }, + { + "value": "cache_hit_rate", + "label": "Cache hit rate" + } + ] + }, + { + "label": "Performance (31 days max)", + "options": [ + { + "value": "p50_latency", + "label": "Latency p50 (ms)" + }, + { + "value": "p90_latency", + "label": "Latency p90 (ms)" + }, + { + "value": "p50_total_time_to_first_token", + "label": "Time to first token p50 (ms)" + }, + { + "value": "p90_total_time_to_first_token", + "label": "Time to first token p90 (ms)" + }, + { + "value": "p50_throughput", + "label": "Throughput p50 (tokens/s)" + } + ] + } + ] + } }, { - "type": "choiceChips", + "type": "autocomplete", "name": "groupBy", "label": "Group by", - "options": [ - { - "value": "model", - "label": "Model" - }, - { - "value": "provider", - "label": "Provider (31 days max)" - }, - { - "value": "workspace", - "label": "Workspace" - }, - { - "value": "api_key_id", - "label": "API key" - }, - { - "value": "app", - "label": "App" - } - ] + "help": "Select a dimension, or enter any dimension from [OpenRouter's analytics dimensions](https://openrouter.ai/docs/api/api-reference/analytics/get-available-analytics-metrics-and-dimensions). Request details are limited to 31 days.", + "allowCustomValues": true, + "isMulti": false, + "isClearable": true, + "data": { + "source": "fixed", + "values": [ + { + "label": "Any timeframe", + "options": [ + { + "value": "model", + "label": "Model" + }, + { + "value": "workspace", + "label": "Workspace" + }, + { + "value": "api_key_id", + "label": "API key" + }, + { + "value": "app", + "label": "App" + }, + { + "value": "user", + "label": "User" + } + ] + }, + { + "label": "Request details (31 days max)", + "options": [ + { + "value": "provider", + "label": "Provider" + }, + { + "value": "origin", + "label": "Origin" + }, + { + "value": "country", + "label": "Country" + }, + { + "value": "finish_reason", + "label": "Finish reason" + }, + { + "value": "streamed", + "label": "Streamed" + } + ] + } + ] + } }, { "type": "objects", @@ -95,6 +181,7 @@ "type": "oneOf", "values": [ "Model", + "Provider", "Workspace", "API Key" ] diff --git a/plugins/OpenRouter/v1/defaultContent/manifest.json b/plugins/OpenRouter/v1/defaultContent/manifest.json index 3116f412..12683324 100644 --- a/plugins/OpenRouter/v1/defaultContent/manifest.json +++ b/plugins/OpenRouter/v1/defaultContent/manifest.json @@ -12,6 +12,10 @@ "name": "model", "type": "dashboard" }, + { + "name": "provider", + "type": "dashboard" + }, { "name": "workspace", "type": "dashboard" diff --git a/plugins/OpenRouter/v1/defaultContent/model.dash.json b/plugins/OpenRouter/v1/defaultContent/model.dash.json index 505de92b..e38ab5a1 100644 --- a/plugins/OpenRouter/v1/defaultContent/model.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/model.dash.json @@ -279,9 +279,312 @@ } }, { - "i": "21bb9221-3105-46cb-a988-b677dafef3e9", + "i": "7a60262f-2a6e-4c04-90c8-d81015d2ba20", "x": 0, "y": 4, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Provider Status", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[modelProviders]}}", + "name": "modelProviders", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + }, + "variables": [ + "{{variables.[Model]}}" + ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "statusGrid", + "encoding": { + "label": { + "field": "tag", + "type": "nominal", + "title": "Endpoint" + }, + "status": { + "field": "health", + "type": "nominal", + "title": "Status" + }, + "value": { + "field": "uptime_last_5m", + "type": "quantitative", + "title": "Uptime (5m)" + } + }, + "style": { + "statusGridColumnsPerRow": 4 + } + } + }, + "visualisation": { + "type": "data-stream-blocks", + "config": { + "data-stream-blocks": { + "labelColumn": "tag", + "stateColumn": "health", + "sublabel": "uptime_last_5m", + "linkColumn": "none", + "columns": 4 + } + } + } + } + }, + { + "i": "2abeda4d-5ec9-4f59-8523-910763ad6575", + "x": 2, + "y": 4, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Provider Price Vs Throughput", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[modelProviders]}}", + "name": "modelProviders", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + }, + "variables": [ + "{{variables.[Model]}}" + ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "scatter", + "encoding": { + "x": { + "field": "outputPrice", + "type": "quantitative", + "title": "Output Price ($/1M Tokens)" + }, + "y": { + "field": "throughput_last_30m.p50", + "type": "quantitative", + "title": "Throughput (Tokens/s)" + }, + "color": { + "field": "provider_name", + "type": "nominal", + "title": "Provider" + } + }, + "style": { + "legendPosition": "none", + "cartesianShowXAxisTitle": true, + "cartesianXAxisTitle": "Output Price ($/1M Tokens)", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Throughput (Tokens/s)" + } + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "provider_name", + "tag", + "outputPrice", + "throughput_last_30m.p50" + ], + "hiddenColumns": [ + "health", + "quantization", + "context_length", + "inputPrice", + "cacheReadPrice", + "uptime_last_5m", + "uptime_last_30m", + "uptime_last_1d", + "latency_last_30m.p50", + "latency_last_30m.p90", + "model_name", + "pricing.prompt", + "pricing.completion", + "status", + "pricing.input_cache_read", + "latency_last_30m.p75", + "latency_last_30m.p99" + ] + } + } + } + } + }, + { + "i": "0e72d465-f7f3-42ad-8a6f-2b7a91a8850a", + "x": 0, + "y": 7, + "w": 2, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Latency p50 (Last 30 Days)", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "p50_latency", + "filterObjects": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": "milliseconds" + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Latency p50", + "showYAxisLabel": true, + "showTrendLine": false + } + } + }, + "timeframe": "last30days" + } + }, + { + "i": "0fe94a89-e194-4ce1-bf03-3495bec434cd", + "x": 2, + "y": 7, + "w": 2, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Time To First Token p50 (Last 30 Days)", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "p50_total_time_to_first_token", + "filterObjects": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": "milliseconds" + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Time To First Token p50", + "showYAxisLabel": true, + "showTrendLine": false + } + } + }, + "timeframe": "last30days" + } + }, + { + "i": "21bb9221-3105-46cb-a988-b677dafef3e9", + "x": 0, + "y": 9, "w": 4, "h": 3, "moved": false, @@ -323,11 +626,14 @@ "transpose": false, "columnOrder": [ "provider_name", + "health", "tag", "quantization", "context_length", "inputPrice", "outputPrice", + "cacheReadPrice", + "uptime_last_5m", "uptime_last_30m", "uptime_last_1d", "latency_last_30m.p50", @@ -337,7 +643,9 @@ "hiddenColumns": [ "model_name", "pricing.prompt", - "pricing.completion" + "pricing.completion", + "status", + "pricing.input_cache_read" ] } } @@ -347,7 +655,7 @@ { "i": "c4ac55e8-5333-45eb-ace6-c5e7aad15b6f", "x": 0, - "y": 7, + "y": 12, "w": 2, "h": 3, "moved": false, @@ -415,7 +723,7 @@ { "i": "e81a0ea0-b5fd-4021-8737-2c50997ab7a4", "x": 2, - "y": 7, + "y": 12, "w": 2, "h": 3, "moved": false, @@ -482,7 +790,7 @@ { "i": "041f6c43-1bf6-4e38-80bb-ca48e03ab8f0", "x": 0, - "y": 10, + "y": 15, "w": 4, "h": 7, "config": { diff --git a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json index cde0ad2c..27c45fbd 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json @@ -312,6 +312,16 @@ "id": "{{dataStreams.[rankings]}}", "name": "rankings", "pluginConfigId": "{{configId}}", + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "model_permaslug", + "operation": "notequals", + "value": "other" + } + ] + }, "group": { "by": [ [ @@ -399,6 +409,16 @@ "id": "{{dataStreams.[rankings]}}", "name": "rankings", "pluginConfigId": "{{configId}}", + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "model_permaslug", + "operation": "notequals", + "value": "other" + } + ] + }, "group": { "by": [ [ @@ -455,10 +475,395 @@ } }, { - "i": "56d4d9f8-9cc4-4852-9306-2997d9e87000", + "i": "071c926f-0d3c-4192-962f-d191aa03dcbc", "x": 0, "y": 10, "w": 6, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Top Models On OpenRouter By Week (This Year)", + "description": "", + "timeframe": "thisYear", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[rankings]}}", + "name": "rankings", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "period": "week", + "top": 10 + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Week" + }, + "y": { + "field": "total_tokens", + "type": "quantitative", + "aggregate": "sum", + "title": "Tokens" + }, + "series": { + "field": "model_permaslug", + "type": "nominal", + "title": "Model" + } + }, + "style": { + "cartesianStack": "on", + "legendPosition": "bottom" + } + } + }, + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "total_tokens" + ], + "seriesColumn": "model_permaslug", + "showLegend": true, + "legendPosition": "bottom", + "yAxisLabel": "Tokens", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "51dfb910-bec4-45ea-b9a2-683c1574f00f", + "x": 0, + "y": 14, + "w": 4, + "h": 5, + "config": { + "_type": "tile/data-stream", + "title": "Tasks On OpenRouter", + "description": "By Share Of Requests", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[taskClassifications]}}", + "name": "taskClassifications", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "usageShare", + "desc" + ] + ] + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "treemap", + "encoding": { + "parent": { + "field": "category", + "type": "nominal", + "title": "Category" + }, + "label": { + "field": "task", + "type": "nominal", + "title": "Task" + }, + "value": { + "field": "usageShare", + "type": "quantitative", + "title": "Request Share" + } + } + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "task", + "yAxisData": [ + "usageShare" + ], + "xAxisGroup": "none", + "xAxisLabel": "Task", + "yAxisLabel": "Request Share", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "0472205a-cff6-442b-93ec-2b063b6aa157", + "x": 4, + "y": 14, + "w": 2, + "h": 5, + "config": { + "_type": "tile/data-stream", + "title": "Request Share By Task Category", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[taskClassifications]}}", + "name": "taskClassifications", + "pluginConfigId": "{{configId}}", + "group": { + "by": [ + [ + "category", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "usageShare" + ] + } + ] + }, + "sort": { + "by": [ + [ + "usageShare_sum", + "desc" + ] + ] + }, + "metadata": [ + { + "name": "usageShare_sum", + "shape": [ + "percent", + { + "asZeroToOne": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "pie", + "encoding": { + "label": { + "field": "category_uniqueValues", + "type": "nominal", + "title": "Category" + }, + "value": { + "field": "usageShare_sum", + "type": "quantitative", + "title": "Request Share" + } + }, + "style": { + "pieInnerRadius": 55, + "legendPosition": "bottom" + } + } + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "usageShare_sum", + "labelColumn": "category_uniqueValues", + "hideCenterValue": false, + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "i": "a3384d79-77a8-49c0-bb08-62d6744303e1", + "x": 0, + "y": 19, + "w": 4, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Top Model By Task", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[taskClassifications]}}", + "name": "taskClassifications", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "usageShare", + "desc" + ] + ] + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "task", + "category", + "usageShare", + "topModel", + "topModelUsageShare", + "asOf" + ], + "hiddenColumns": [ + "tag", + "tokenShare" + ] + } + } + } + } + }, + { + "i": "488c8839-874c-4a1e-a32f-5318792e4b68", + "x": 4, + "y": 19, + "w": 2, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Providers By Headquarters", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[providers]}}", + "name": "providers", + "pluginConfigId": "{{configId}}", + "group": { + "by": [ + [ + "headquarters", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "count" + } + ] + }, + "sort": { + "by": [ + [ + "count", + "desc" + ] + ] + }, + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "headquarters", + "operation": "notempty" + } + ] + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "pie", + "encoding": { + "label": { + "field": "headquarters_uniqueValues", + "type": "nominal", + "title": "Headquarters" + }, + "value": { + "field": "count", + "type": "quantitative", + "title": "Providers" + } + }, + "style": { + "legendPosition": "bottom" + } + } + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "count", + "labelColumn": "headquarters_uniqueValues", + "hideCenterValue": false, + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "i": "56d4d9f8-9cc4-4852-9306-2997d9e87000", + "x": 0, + "y": 23, + "w": 6, "h": 8, "config": { "_type": "tile/data-stream", @@ -609,7 +1014,7 @@ { "i": "679f0166-77c4-47c1-9ae5-7e4c1721f718", "x": 0, - "y": 18, + "y": 31, "w": 6, "h": 1, "config": { @@ -618,7 +1023,7 @@ "description": "", "visualisation": { "config": { - "content": "Benchmark data from Artificial Analysis, Design Arena, and OpenRouter, via OpenRouter \nSource: [OpenRouter](https://openrouter.ai/rankings), licensed CC BY 4.0", + "content": "Benchmark data from Artificial Analysis, Design Arena, and OpenRouter, via OpenRouter. Task data is as of the **As Of** date shown in the Top Model By Task table. \nSource: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)), licensed CC BY 4.0", "fontSize": 12, "align": "left", "autoSize": true diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json index 833f63a2..9f3136ab 100644 --- a/plugins/OpenRouter/v1/defaultContent/overview.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -334,9 +334,104 @@ } }, { - "i": "55c667d9-63af-4556-962d-87a6325df855", + "i": "89d9ae8b-e700-4c31-813a-8380d9511fbb", "x": 0, "y": 4, + "w": 4, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend By Model Over Time", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "groupBy": "model" + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "aggregate": "sum", + "title": "Spend ($)" + }, + "series": { + "field": "group", + "type": "nominal", + "title": "Model" + } + }, + "style": { + "cartesianStack": "on", + "legendPosition": "bottom" + } + } + }, + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "group", + "showLegend": true, + "legendPosition": "bottom", + "yAxisLabel": "Spend ($)", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "55c667d9-63af-4556-962d-87a6325df855", + "x": 0, + "y": 7, "w": 2, "h": 3, "moved": false, @@ -430,7 +525,7 @@ { "i": "7c94001a-4946-45e0-8d08-08f2b89153ef", "x": 2, - "y": 4, + "y": 7, "w": 2, "h": 3, "moved": false, @@ -507,10 +602,200 @@ } } }, + { + "i": "da913ee7-05b2-4ce2-b11a-64b91f9d1537", + "x": 0, + "y": 10, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend By Provider (Last 30 Days)", + "description": "", + "timeframe": "last30days", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "groupBy": "provider" + }, + "group": { + "by": [ + [ + "group", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "value" + ] + } + ] + }, + "sort": { + "by": [ + [ + "value_sum", + "desc" + ] + ] + }, + "metadata": [ + { + "name": "value_sum", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "pie", + "encoding": { + "label": { + "field": "group_uniqueValues", + "type": "nominal", + "title": "Provider" + }, + "value": { + "field": "value_sum", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "pieInnerRadius": 55, + "legendPosition": "right" + } + } + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "value_sum", + "labelColumn": "group_uniqueValues", + "hideCenterValue": false, + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "i": "3bf76eb4-561e-4bae-a971-ba12c6c6d216", + "x": 2, + "y": 10, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Latency p50 By Model (Last 7 Days)", + "description": "", + "timeframe": "last7days", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "p50_latency", + "groupBy": "model" + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": "milliseconds" + }, + { + "pattern": ".*" + } + ] + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Latency p50" + }, + "series": { + "field": "group", + "type": "nominal", + "title": "Model" + } + }, + "style": { + "legendPosition": "bottom" + } + } + }, + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "group", + "showLegend": true, + "legendPosition": "bottom", + "yAxisLabel": "Latency p50", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, { "i": "d596864a-98e3-466c-ba4d-cd59ee6f7f01", "x": 0, - "y": 7, + "y": 13, "w": 4, "h": 3, "moved": false, diff --git a/plugins/OpenRouter/v1/defaultContent/provider.dash.json b/plugins/OpenRouter/v1/defaultContent/provider.dash.json new file mode 100644 index 00000000..b27feef3 --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/provider.dash.json @@ -0,0 +1,469 @@ +{ + "name": "Provider", + "schemaVersion": "1.5", + "timeframe": "last30days", + "variables": [ + "{{variables.[Provider]}}" + ], + "dashboard": { + "_type": "layout/grid", + "columns": 4, + "version": 1, + "contents": [ + { + "i": "351a6639-f310-42e3-91c2-7beb54aefd69", + "x": 0, + "y": 0, + "w": 1, + "h": 4, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Properties", + "description": "", + "timeframe": "none", + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Providers]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Provider]}}" + }, + "variables": [ + "{{variables.[Provider]}}" + ], + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": true, + "columnOrder": [ + "name", + "headquarters", + "datacenters", + "status_page_url" + ], + "hiddenColumns": [ + "id", + "links", + "type", + "label", + "sourceId", + "sourceType", + "sourceName", + "rawId" + ] + } + } + } + } + }, + { + "i": "39e8dd4a-e54f-4e45-bd83-b0138ca98c1a", + "x": 1, + "y": 0, + "w": 3, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Spend (Last 30 Days)", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "filterObjects": { + "scope": "{{scopes.[Providers]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Provider]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Provider]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Spend ($)", + "showYAxisLabel": true, + "showTrendLine": false + } + } + }, + "timeframe": "last30days" + } + }, + { + "i": "19df04e1-287e-4e3b-a37c-a89fafc1cd76", + "x": 1, + "y": 2, + "w": 3, + "h": 2, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Requests (Last 30 Days)", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "request_count", + "filterObjects": { + "scope": "{{scopes.[Providers]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Provider]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Provider]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Requests", + "showYAxisLabel": true, + "showTrendLine": false + } + } + }, + "timeframe": "last30days" + } + }, + { + "i": "35d28afb-e7eb-4170-8a40-b5e6e2f2b34d", + "x": 0, + "y": 4, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Latency p50 (Last 30 Days)", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "p50_latency", + "filterObjects": { + "scope": "{{scopes.[Providers]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Provider]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Provider]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Latency (ms)", + "showYAxisLabel": true, + "showTrendLine": false + } + } + }, + "timeframe": "last30days" + } + }, + { + "i": "26bc4fa7-e072-4441-ada9-90dab73e0946", + "x": 2, + "y": 4, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Throughput p50 (Last 30 Days)", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "p50_throughput", + "filterObjects": { + "scope": "{{scopes.[Providers]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Provider]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "number", + { + "decimalPlaces": 1 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Provider]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "none", + "showLegend": false, + "legendPosition": "bottom", + "yAxisLabel": "Tokens/s", + "showYAxisLabel": true, + "showTrendLine": false + } + } + }, + "timeframe": "last30days" + } + }, + { + "i": "2c909c6a-a40e-4f6f-9ac9-2487d77f470a", + "x": 0, + "y": 7, + "w": 4, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend By Model On This Provider (Last 30 Days)", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "groupBy": "model", + "filterObjects": { + "scope": "{{scopes.[Providers]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Provider]}}" + } + }, + "group": { + "by": [ + [ + "group", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "value" + ] + } + ] + }, + "sort": { + "by": [ + [ + "value_sum", + "desc" + ] + ], + "top": 10 + }, + "metadata": [ + { + "name": "value_sum", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Provider]}}" + ], + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "group_uniqueValues", + "yAxisData": [ + "value_sum" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Spend ($)", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + }, + "timeframe": "last30days" + } + } + ] + } +} diff --git a/plugins/OpenRouter/v1/defaultContent/scopes.json b/plugins/OpenRouter/v1/defaultContent/scopes.json index 59998e4e..4a701224 100644 --- a/plugins/OpenRouter/v1/defaultContent/scopes.json +++ b/plugins/OpenRouter/v1/defaultContent/scopes.json @@ -16,6 +16,23 @@ "allowMultipleSelection": false } }, + { + "name": "Providers", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Provider" + ] + } + }, + "variable": { + "name": "Provider", + "type": "object", + "default": "none", + "allowMultipleSelection": false + } + }, { "name": "Workspaces", "matches": { diff --git a/plugins/OpenRouter/v1/docs/README.md b/plugins/OpenRouter/v1/docs/README.md index e78a2d47..8957634e 100644 --- a/plugins/OpenRouter/v1/docs/README.md +++ b/plugins/OpenRouter/v1/docs/README.md @@ -26,48 +26,54 @@ On save, the plugin checks the key in two steps. An invalid, disabled, or expire ## What this plugin monitors -- **Spend and usage** — spend, requests, and tokens over time, broken down by model, provider, workspace, API key, or app. +- **Spend and usage** — spend, requests, tokens, latency, and throughput over time, broken down by model, provider, workspace, API key, or app. - **API keys** — spend today, this week, and this month for each key, against its credit limit, plus an expiry warning. - **Account credits** — credits purchased, used, and remaining. - **Model comparison** — pricing, context length, modalities, and Artificial Analysis benchmark scores for every model on OpenRouter. -- **Provider performance** — price, uptime, latency, and throughput for each provider serving a model. -- **Benchmarks and rankings** — scores from Artificial Analysis, Design Arena, and OpenRouter's own evals, and OpenRouter's public leaderboard of the most used models. +- **Providers** — every provider serving models on OpenRouter, with headquarters and datacenter locations. +- **Provider performance** — current status, price, uptime, latency, and throughput for each provider serving a model. +- **Benchmarks and rankings** — scores from Artificial Analysis, Design Arena, and OpenRouter's own evals, OpenRouter's public leaderboard of the most used models, and what share of OpenRouter traffic goes to each kind of task. -The out-of-the-box dashboards include an account **Overview**, a **Model Comparison** dashboard (which works with a standard API key), and a perspective for each **Model**, **Workspace**, and **API Key** (spend tiles need a management key). +The out-of-the-box dashboards include an account **Overview**, a **Model Comparison** dashboard (which works with a standard API key), and a perspective for each **Model**, **Provider**, **Workspace**, and **API Key** (spend tiles need a management key). The Model perspective includes a current view of each provider's status, price, and throughput. ## Data streams -- **Usage** — spend, requests, tokens, or cache hit rate over time, optionally grouped by model, provider, workspace, API key, or app, and optionally filtered to selected objects. Account-wide. Management key only. +- **Usage** — spend, requests, tokens, cache hit rate, latency, time to first token, or throughput over time, optionally grouped by model, provider, workspace, API key, app, user, origin, country, finish reason, or streamed, and optionally filtered to selected models, providers, workspaces, or API keys. Any other OpenRouter analytics metric or dimension can be typed in. Account-wide. Management key only. - **API Keys** — API keys in a workspace with spend and limits. Per workspace. Management key only. - **Workspaces** — workspaces in the account. Account-wide. Management key only. - **Credits** — credits purchased, used, and remaining. Account-wide. - **Models** — every model on OpenRouter with pricing ($ per 1M tokens), context length, modalities, and Artificial Analysis index scores, optionally filtered to selected models. -- **Model Providers** — providers serving a model, with price, uptime, latency, and throughput over the last 30 minutes. Per model. +- **Providers** — every provider on OpenRouter, with headquarters, datacenter countries, and status, privacy, and terms links. +- **Model Providers** — providers serving a model, with status, price (including cache reads), uptime over the last 5 minutes, 30 minutes, and day, and latency (p50 to p99) and throughput over the last 30 minutes. Per model. - **Benchmarks** — model benchmark scores from Artificial Analysis, Design Arena, and OpenRouter evals, one row per model and benchmark. -- **Model Rankings** — the most used models across all OpenRouter traffic, by tokens processed per day. +- **Model Rankings** — the most used models across all OpenRouter traffic, by tokens processed per day, week, or month. Optionally keep only the top models and combine the rest into **Other**. +- **Task Classifications** — share of all OpenRouter requests and tokens by task type (such as code generation or summarization) over the last 7 days, grouped into Code, Data, Agent, and General, with each task's most used model. ## What gets indexed | Object type | API source | Represents | | ------------- | ------------------------------- | ------------------------------------------------------------------------ | | **Model** | `GET /models` | A model available through OpenRouter, such as `anthropic/claude-sonnet-4.5`. | +| **Provider** | `GET /providers` | A provider serving models through OpenRouter, such as `Groq`. | | **Workspace** | `GET /workspaces` | A workspace in the account. Management key only. | | **API Key** | `GET /keys` (per workspace) | An API key in a workspace, including its expiry date. Management key only. | **Relationships:** each API Key records the ID of its Workspace. -With a standard API key, only models are imported; the workspace and API key import steps are skipped with a warning. +With a standard API key, only models and providers are imported; the workspace and API key import steps are skipped with a warning. ## Known limitations -- **Benchmarks and rankings refresh at most once a day** — OpenRouter limits these endpoints to 500 requests a day per account, so the plugin caches them for 24 hours. -- **Rankings licence** — rankings data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). The **Model Comparison** dashboard credits it already; keep "Source: OpenRouter (openrouter.ai/rankings)" on any dashboard you build from it. +- **Benchmarks, rankings, and task classifications refresh at most once a day** — OpenRouter limits these endpoints to 500 requests a day per account, so the plugin caches them for 24 hours. +- **Rankings licence** — rankings and task classification data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). The **Model Comparison** dashboard credits it already; keep "Source: OpenRouter (openrouter.ai/rankings)" on any dashboard you build from it, adding "as of" and the **As Of** date for task classifications. - **Not every model has benchmark scores** — Artificial Analysis scores cover fewer than half of models; the rest show blank. -- **Usage history** can be queried up to a year back. Grouping usage by **Provider** is limited to 31 days; longer timeframes return an error. +- **Usage history** can be queried up to a year back. Latency, time to first token, throughput, and grouping by **Provider**, **Origin**, **Country**, **Finish reason**, or **Streamed** are limited to 31 days; longer timeframes return an error. - **Usage timeframes start at 12 hours** — data is bucketed by hour, day, week, or month to suit the range; periods with no activity are left out rather than shown as zero. - **Large usage queries may be truncated** — the stream warns when this happens; narrow the timeframe or filter to specific objects. - **Model rankings start at 7 days** — the data is daily. -- **Provider performance is a live snapshot** — it covers the last 30 minutes, with no history beyond what SquaredUp records. -- **Filtering usage needs imported objects** — pick workspaces and API keys in **Filter to objects (optional)**; grouped results show their names. +- **Task classifications cover the last 7 days only** — they come from a sample of OpenRouter traffic, so only shares are available, not request or token counts. +- **Provider status is inferred** — OpenRouter doesn't document its endpoint status codes. The plugin shows `0` as healthy, `-1` to `-3` as warning, and anything lower as error, which matches the uptime OpenRouter reports for those endpoints; check **Uptime (5m)** alongside it. +- **Provider performance is a live snapshot** — it covers the last 5 minutes to 1 day, with no history beyond what SquaredUp records. +- **Filtering usage needs imported objects** — pick models, providers, workspaces, or API keys in **Filter to objects (optional)**; grouped results show their names. Filtering by provider is limited to 31 days. - **Organization members aren't imported.** - **Read-only** — the plugin never creates, modifies, or deletes anything in OpenRouter. diff --git a/plugins/OpenRouter/v1/indexDefinitions/default.json b/plugins/OpenRouter/v1/indexDefinitions/default.json index 9b180b54..00b5b253 100644 --- a/plugins/OpenRouter/v1/indexDefinitions/default.json +++ b/plugins/OpenRouter/v1/indexDefinitions/default.json @@ -15,6 +15,21 @@ "properties": ["author", "canonical_slug", "context_length"] } }, + { + "name": "providers", + "dataStream": { + "name": "providers" + }, + "timeframe": "none", + "objectMapping": { + "id": "slug", + "name": "name", + "type": { + "value": "Provider" + }, + "properties": ["headquarters", "datacenters", "status_page_url"] + } + }, { "name": "workspaces", "dataStream": { diff --git a/plugins/OpenRouter/v1/metadata.json b/plugins/OpenRouter/v1/metadata.json index b5f37d33..2137586b 100644 --- a/plugins/OpenRouter/v1/metadata.json +++ b/plugins/OpenRouter/v1/metadata.json @@ -23,7 +23,7 @@ "tokens", "billing" ], - "objectTypes": ["Model", "Workspace", "API Key"], + "objectTypes": ["Model", "Provider", "Workspace", "API Key"], "links": [ { "category": "documentation", From 3617ce34cbd226d6b899e7ee7c2b195b7272e27a Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Fri, 2 Oct 2026 19:45:55 +0100 Subject: [PATCH 03/16] Add OpenRouter public datasets and an Ecosystem dashboard Squashed from: - Add citation columns to OpenRouter public datasets and name provider countries - Add OpenRouter session cost stream and tiles - Rank OpenRouter top models per period to match OpenRouter's chart - Use a linear axis for OpenRouter session cost by harness - Add OpenRouter app rankings stream and tiles - Add trending apps treemap to the OpenRouter overview - Add an OpenRouter Ecosystem dashboard for platform-wide data - Add use case, language, modality, and context filters to OpenRouter rankings Co-Authored-By: Claude Opus 5.5 --- .../v1/dataStreams/appRankings.json | 238 ++++++++++ .../OpenRouter/v1/dataStreams/benchmarks.json | 12 + .../OpenRouter/v1/dataStreams/rankings.json | 211 ++++++--- .../v1/dataStreams/scripts/appRankings.js | 5 + .../v1/dataStreams/scripts/benchmarks.js | 10 + .../v1/dataStreams/scripts/providers.js | 22 +- .../v1/dataStreams/scripts/rankings.js | 50 +- .../v1/dataStreams/scripts/sessionCost.js | 24 + .../scripts/taskClassifications.js | 4 +- .../v1/dataStreams/sessionCost.json | 178 ++++++++ .../v1/dataStreams/taskClassifications.json | 9 +- .../v1/defaultContent/ecosystem.dash.json | 429 ++++++++++++++++++ .../v1/defaultContent/manifest.json | 4 + .../v1/defaultContent/model.dash.json | 119 ++++- .../defaultContent/modelComparison.dash.json | 290 +++--------- .../v1/defaultContent/overview.dash.json | 4 +- plugins/OpenRouter/v1/docs/README.md | 16 +- 17 files changed, 1297 insertions(+), 328 deletions(-) create mode 100644 plugins/OpenRouter/v1/dataStreams/appRankings.json create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/appRankings.js create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/sessionCost.js create mode 100644 plugins/OpenRouter/v1/dataStreams/sessionCost.json create mode 100644 plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json diff --git a/plugins/OpenRouter/v1/dataStreams/appRankings.json b/plugins/OpenRouter/v1/dataStreams/appRankings.json new file mode 100644 index 00000000..0fb54bbd --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/appRankings.json @@ -0,0 +1,238 @@ +{ + "name": "appRankings", + "displayName": "App Rankings", + "description": "Top public apps on OpenRouter by tokens processed, most popular or trending (from OpenRouter's public apps marketplace)", + "tags": [ + "Usage" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "datasets/app-rankings", + "paging": { + "mode": "none" + }, + "getArgs": [ + { + "key": "start_date", + "value": "{{timeframe.start.substring(0, 10)}}" + }, + { + "key": "end_date", + "value": "{{timeframe.end.substring(0, 10)}}" + }, + { + "key": "sort", + "value": "{{ typeof rankBy !== 'undefined' && rankBy ? rankBy : 'popular' }}" + }, + { + "key": "category", + "value": "{{ (c => ['coding', 'creative', 'productivity', 'entertainment'].includes(c) ? c : null)(typeof category !== 'undefined' ? category : null) }}" + }, + { + "key": "subcategory", + "value": "{{ (c => c && !['coding', 'creative', 'productivity', 'entertainment'].includes(c) ? c : null)(typeof category !== 'undefined' ? category : null) }}" + }, + { + "key": "limit", + "value": "100" + } + ], + "headers": [], + "postRequestScript": "appRankings.js" + }, + "matches": "none", + "metadata": [ + { + "name": "rank", + "displayName": "Rank", + "shape": "number" + }, + { + "name": "app_id", + "displayName": "App ID", + "shape": "string", + "visible": false + }, + { + "name": "app_name", + "displayName": "App", + "shape": "string", + "role": "label" + }, + { + "name": "total_tokens", + "displayName": "Total Tokens", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ], + "role": "value" + }, + { + "name": "total_requests", + "displayName": "Total Requests", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "name": "asOf", + "displayName": "As Of", + "shape": "date", + "visible": false + }, + { + "name": "citation", + "displayName": "Citation", + "shape": "string", + "visible": false + } + ], + "timeframes": [ + "last7days", + "last30days", + "thisMonth", + "lastMonth", + "thisQuarter", + "lastQuarter", + "thisYear", + "lastYear" + ], + "options": { + "minimumRequestIntervalSeconds": 86400 + }, + "ui": [ + { + "type": "choiceChips", + "name": "rankBy", + "label": "Rank by", + "defaultValue": "popular", + "options": [ + { + "value": "popular", + "label": "Most tokens" + }, + { + "value": "trending", + "label": "Trending" + } + ], + "help": "**Trending** ranks apps by token growth over the previous three periods of the same length, so it may return fewer apps" + }, + { + "type": "autocomplete", + "name": "category", + "label": "Category", + "isClearable": true, + "data": { + "source": "fixed", + "values": [ + { + "label": "Coding", + "options": [ + { + "value": "coding", + "label": "All coding" + }, + { + "value": "cli-agent", + "label": "CLI agents" + }, + { + "value": "ide-extension", + "label": "IDE extensions" + }, + { + "value": "cloud-agent", + "label": "Cloud agents" + }, + { + "value": "programming-app", + "label": "Programming apps" + }, + { + "value": "native-app-builder", + "label": "Native app builders" + } + ] + }, + { + "label": "Creative", + "options": [ + { + "value": "creative", + "label": "All creative" + }, + { + "value": "creative-writing", + "label": "Creative writing" + }, + { + "value": "video-gen", + "label": "Video generation" + }, + { + "value": "image-gen", + "label": "Image generation" + }, + { + "value": "audio-gen", + "label": "Audio generation" + } + ] + }, + { + "label": "Productivity", + "options": [ + { + "value": "productivity", + "label": "All productivity" + }, + { + "value": "writing-assistant", + "label": "Writing assistants" + }, + { + "value": "general-chat", + "label": "General chat" + }, + { + "value": "personal-agent", + "label": "Personal agents" + }, + { + "value": "legal", + "label": "Legal" + } + ] + }, + { + "label": "Entertainment", + "options": [ + { + "value": "entertainment", + "label": "All entertainment" + }, + { + "value": "roleplay", + "label": "Roleplay" + }, + { + "value": "game", + "label": "Games" + } + ] + } + ] + } + } + ] +} diff --git a/plugins/OpenRouter/v1/dataStreams/benchmarks.json b/plugins/OpenRouter/v1/dataStreams/benchmarks.json index 8498d983..ba4a5d5f 100644 --- a/plugins/OpenRouter/v1/dataStreams/benchmarks.json +++ b/plugins/OpenRouter/v1/dataStreams/benchmarks.json @@ -105,6 +105,18 @@ "name": "lastRun", "displayName": "Last Run", "shape": "date" + }, + { + "name": "asOf", + "displayName": "As Of", + "shape": "date", + "visible": false + }, + { + "name": "citation", + "displayName": "Citation", + "shape": "string", + "visible": false } ], "options": { diff --git a/plugins/OpenRouter/v1/dataStreams/rankings.json b/plugins/OpenRouter/v1/dataStreams/rankings.json index 87bcee95..20cd309a 100644 --- a/plugins/OpenRouter/v1/dataStreams/rankings.json +++ b/plugins/OpenRouter/v1/dataStreams/rankings.json @@ -24,11 +24,23 @@ }, { "key": "period", - "value": "{{period || null}}" + "value": "{{ (p => /^(category|language):/.test((typeof segment !== 'undefined' && segment ? segment : '')) && (!p || p === 'day') ? 'week' : p || null)(typeof period !== 'undefined' ? period : null) }}" }, { "key": "category", - "value": "{{category || null}}" + "value": "{{ (s => s.startsWith('category:') ? s.slice(9) : null)((typeof segment !== 'undefined' && segment ? segment : '')) }}" + }, + { + "key": "language_type", + "value": "{{ (s => s.startsWith('language:') ? s.slice(9) : null)((typeof segment !== 'undefined' && segment ? segment : '')) }}" + }, + { + "key": "modality", + "value": "{{ (s => s.startsWith('modality:') ? s.slice(9) : null)((typeof segment !== 'undefined' && segment ? segment : '')) }}" + }, + { + "key": "context_bucket", + "value": "{{ (s => s.startsWith('context:') ? s.slice(8) : null)((typeof segment !== 'undefined' && segment ? segment : '')) }}" } ], "headers": [], @@ -56,65 +68,138 @@ ] }, { - "type": "choiceChips", - "name": "category", - "label": "Category", - "options": [ - { - "value": "programming", - "label": "Programming" - }, - { - "value": "roleplay", - "label": "Roleplay" - }, - { - "value": "marketing", - "label": "Marketing" - }, - { - "value": "marketing/seo", - "label": "Marketing / SEO" - }, - { - "value": "technology", - "label": "Technology" - }, - { - "value": "science", - "label": "Science" - }, - { - "value": "translation", - "label": "Translation" - }, - { - "value": "legal", - "label": "Legal" - }, - { - "value": "finance", - "label": "Finance" - }, - { - "value": "health", - "label": "Health" - }, - { - "value": "trivia", - "label": "Trivia" - }, - { - "value": "academia", - "label": "Academia" - } - ] + "type": "autocomplete", + "name": "segment", + "label": "Segment (optional)", + "isClearable": true, + "help": "Shows weekly estimates for **Use case** and **Language** segments, so a **Day** period is shown weekly", + "data": { + "source": "fixed", + "values": [ + { + "label": "Use case (estimated)", + "options": [ + { + "value": "category:programming", + "label": "Programming" + }, + { + "value": "category:roleplay", + "label": "Roleplay" + }, + { + "value": "category:marketing", + "label": "Marketing" + }, + { + "value": "category:marketing/seo", + "label": "Marketing / SEO" + }, + { + "value": "category:technology", + "label": "Technology" + }, + { + "value": "category:science", + "label": "Science" + }, + { + "value": "category:translation", + "label": "Translation" + }, + { + "value": "category:legal", + "label": "Legal" + }, + { + "value": "category:finance", + "label": "Finance" + }, + { + "value": "category:health", + "label": "Health" + }, + { + "value": "category:trivia", + "label": "Trivia" + }, + { + "value": "category:academia", + "label": "Academia" + } + ] + }, + { + "label": "Language (estimated)", + "options": [ + { + "value": "language:natural", + "label": "Natural language" + }, + { + "value": "language:programming", + "label": "Programming languages" + } + ] + }, + { + "label": "Modality", + "options": [ + { + "value": "modality:text", + "label": "Text output" + }, + { + "value": "modality:image_output", + "label": "Image output" + }, + { + "value": "modality:image", + "label": "Image input" + }, + { + "value": "modality:audio", + "label": "Audio input" + }, + { + "value": "modality:tool_calling", + "label": "Tool calling" + } + ] + }, + { + "label": "Context length", + "options": [ + { + "value": "context:1K", + "label": "1K tokens" + }, + { + "value": "context:10K", + "label": "10K tokens" + }, + { + "value": "context:100K", + "label": "100K tokens" + }, + { + "value": "context:1M", + "label": "1M tokens" + }, + { + "value": "context:10M", + "label": "10M tokens" + } + ] + } + ] + } }, { "type": "number", "name": "top", - "label": "Top models (optional)", - "help": "Keep the models with the most tokens across the timeframe and combine the rest into **Other**" + "label": "Top models per period (optional)", + "help": "Keep the models with the most tokens in each period and combine the rest into **Other**" } ], "metadata": [ @@ -147,6 +232,18 @@ "thousandsSeparator": true } ] + }, + { + "name": "asOf", + "displayName": "As Of", + "shape": "date", + "visible": false + }, + { + "name": "citation", + "displayName": "Citation", + "shape": "string", + "visible": false } ], "timeframes": [ diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/appRankings.js b/plugins/OpenRouter/v1/dataStreams/scripts/appRankings.js new file mode 100644 index 00000000..b50dfcca --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/appRankings.js @@ -0,0 +1,5 @@ +// Adds the as-of date and citation OpenRouter requires to every row +const asOf = data?.meta?.as_of; +const citation = `Source: OpenRouter (openrouter.ai/apps), as of ${asOf}. Licensed under CC BY 4.0.`; + +result = (data?.data ?? []).map((r) => ({ ...r, asOf, citation })); diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/benchmarks.js b/plugins/OpenRouter/v1/dataStreams/scripts/benchmarks.js index 1b4862e1..742fc9e5 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/benchmarks.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/benchmarks.js @@ -5,6 +5,14 @@ const aaIndices = [ ['agentic_index', 'Agentic Index'], ]; +// Attribution OpenRouter requires per source when republishing scores +const citations = { + 'artificial-analysis': 'Source: Artificial Analysis (artificialanalysis.ai) via OpenRouter (openrouter.ai/rankings). Licensed under CC BY 4.0.', + 'design-arena': 'Source: Design Arena (www.designarena.ai) via OpenRouter (openrouter.ai/rankings). Licensed under CC BY 4.0.', + openrouter: 'Source: OpenRouter evals (openrouter.ai) via OpenRouter (openrouter.ai/rankings). Licensed under CC BY 4.0.', +}; +const asOf = data?.meta?.as_of; + const rows = []; for (const r of data?.data ?? []) { @@ -12,6 +20,8 @@ for (const r of data?.data ?? []) { source: r.source, model: r.model_permaslug, modelName: r.display_name, + asOf, + citation: citations[r.source], }; if (r.source === 'artificial-analysis') { diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/providers.js b/plugins/OpenRouter/v1/dataStreams/scripts/providers.js index 7fe8f5d6..72a8826c 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/providers.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/providers.js @@ -1,3 +1,19 @@ -// Flattens the datacenters country-code array into a comma-separated string -// so it can be shown in tables and indexed as a Provider property. -result = (data?.data ?? []).map((p) => ({ ...p, datacenters: (p.datacenters ?? []).join(', ') })); +// Expands ISO country codes to names (US -> United States) and flattens the +// datacenters array into a comma-separated string, so both read well in tiles +// and can be indexed as Provider properties. +const countries = new Intl.DisplayNames(['en'], { type: 'region' }); + +// Falls back to the raw code if it isn't a valid region code +const country = (code) => { + try { + return code ? countries.of(code) : code; + } catch { + return code; + } +}; + +result = (data?.data ?? []).map((p) => ({ + ...p, + headquarters: country(p.headquarters), + datacenters: (p.datacenters ?? []).map(country).join(', '), +})); diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js b/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js index 440123d9..64d96159 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js @@ -1,41 +1,41 @@ -// Optional `top`: keep the N models with the most tokens across the window and -// merge the rest (including OpenRouter's own `other` row) into one "Other" row -// per date, so stacked charts stay readable. Without it, rows pass through. +// Optional `top`: for each date (day, week, or month), keep that period's N +// biggest models and merge the rest (including OpenRouter's own `other` row) +// into one "Other" row, matching how openrouter.ai/rankings charts its top +// models. Without it, rows pass through. Every row then gets the as-of date +// and citation OpenRouter requires. const rows = data?.data ?? []; const top = Number(context.config.top); if (!(top > 0)) { result = rows; } else { - const totals = {}; + const byDate = {}; for (const r of rows) { - if (r.model_permaslug === 'other') continue; - totals[r.model_permaslug] = (totals[r.model_permaslug] ?? 0) + Number(r.total_tokens); + const period = byDate[r.date] ?? (byDate[r.date] = []); + period.push({ ...r, total_tokens: Number(r.total_tokens) }); } - const kept = new Set( - Object.entries(totals) - .sort(([, a], [, b]) => b - a) - .slice(0, top) - .map(([model]) => model), - ); + result = []; - const otherByDate = {}; + for (const [date, periodRows] of Object.entries(byDate).sort(([a], [b]) => a.localeCompare(b))) { + const ranked = periodRows + .filter((r) => r.model_permaslug !== 'other') + .sort((a, b) => b.total_tokens - a.total_tokens); - result = []; + const kept = ranked.slice(0, top); + const keptModels = new Set(kept.map((r) => r.model_permaslug)); + const other = periodRows + .filter((r) => !keptModels.has(r.model_permaslug)) + .reduce((sum, r) => sum + r.total_tokens, 0); - for (const r of rows) { - if (kept.has(r.model_permaslug)) { - result.push({ ...r, total_tokens: Number(r.total_tokens) }); - } else { - otherByDate[r.date] = (otherByDate[r.date] ?? 0) + Number(r.total_tokens); - } - } + result.push(...kept); - for (const [date, total_tokens] of Object.entries(otherByDate)) { - result.push({ date, model_permaslug: 'Other', total_tokens }); + if (other > 0) result.push({ date, model_permaslug: 'Other', total_tokens: other }); } - - result.sort((a, b) => a.date.localeCompare(b.date)); } + +// Attribution OpenRouter requires when republishing this dataset +const asOf = data?.meta?.as_of; +const citation = `Source: OpenRouter (openrouter.ai/rankings), as of ${asOf}. Licensed under CC BY 4.0.`; +result = result.map((r) => ({ ...r, asOf, citation })); diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/sessionCost.js b/plugins/OpenRouter/v1/dataStreams/scripts/sessionCost.js new file mode 100644 index 00000000..358494e2 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/sessionCost.js @@ -0,0 +1,24 @@ +// Labels session lengths (with a sort order, as "10-49 turns" sorts before +// "2-9 turns" alphabetically), applies the optional `models` object picker, +// and adds the as-of date and citation OpenRouter requires to every row. +const lengths = { + '1-turn': ['1 turn', 1], + '2-9-turns': ['2–9 turns', 2], + '10-49-turns': ['10–49 turns', 3], + '50-plus-turns': ['50+ turns', 4], +}; + +// Selected Model objects arrive with each property as a single-element array +const unwrap = (v) => (Array.isArray(v) ? v[0] : v); +const slugs = new Set((context.config?.models ?? []).map((o) => unwrap(o.canonical_slug)).filter(Boolean)); + +const asOf = data?.meta?.as_of; +const citation = `Source: OpenRouter (openrouter.ai/rankings), as of ${asOf}. Licensed under CC BY 4.0.`; + +result = (data?.data ?? []) + .filter((r) => !slugs.size || slugs.has(r.model_permaslug)) + .map((r) => { + const [sessionLength, sessionLengthOrder] = lengths[r.turn_range] ?? [r.turn_range, null]; + + return { ...r, sessionLength, sessionLengthOrder, asOf, citation }; + }); diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/taskClassifications.js b/plugins/OpenRouter/v1/dataStreams/scripts/taskClassifications.js index 9011fb73..229a98d9 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/taskClassifications.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/taskClassifications.js @@ -1,5 +1,6 @@ // One row per task classification, with its macro-category label and the -// task's most used model. Shares are fractions (0-1) of sampled traffic. +// task's most used model, plus the as-of date and citation OpenRouter requires. +// Shares are fractions (0-1) of sampled traffic. const body = data?.data ?? {}; const categoryLabels = Object.fromEntries((body.macro_categories ?? []).map((c) => [c.key, c.label])); @@ -16,5 +17,6 @@ result = (body.classifications ?? []).map((c) => { topModel: topModel?.id ?? null, topModelUsageShare: topModel?.tag_usage_share ?? null, asOf: body.as_of, + citation: `Source: OpenRouter (openrouter.ai/rankings), as of ${body.as_of}. Licensed under CC BY 4.0.`, }; }); diff --git a/plugins/OpenRouter/v1/dataStreams/sessionCost.json b/plugins/OpenRouter/v1/dataStreams/sessionCost.json new file mode 100644 index 00000000..efe8a9ca --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/sessionCost.json @@ -0,0 +1,178 @@ +{ + "name": "sessionCost", + "displayName": "Session Cost", + "description": "Median cost of one coding-agent session by harness, model, and session length (from OpenRouter's public rankings, refreshed weekly)", + "tags": [ + "Models", + "Cost" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "get", + "endpointPath": "datasets/session-cost", + "paging": { + "mode": "offset", + "pageSize": { + "realm": "queryArg", + "path": "limit", + "value": "500" + }, + "offset": { + "mode": "row", + "rowCountIn": { + "realm": "payloadArraySize", + "path": "data" + }, + "base": 0 + }, + "out": { + "realm": "queryArg", + "path": "offset" + } + }, + "getArgs": [ + { + "key": "app_slug", + "value": "{{harness || null}}" + }, + { + "key": "turn_range", + "value": "{{turnRange || null}}" + } + ], + "headers": [], + "postRequestScript": "sessionCost.js" + }, + "matches": "none", + "metadata": [ + { + "name": "app_name", + "displayName": "Harness", + "shape": "string", + "role": "label" + }, + { + "name": "app_slug", + "displayName": "Harness ID", + "shape": "string", + "visible": false + }, + { + "name": "model_permaslug", + "displayName": "Model", + "shape": "string" + }, + { + "name": "turn_range", + "displayName": "Session Length ID", + "shape": "string", + "visible": false + }, + { + "name": "sessionLength", + "displayName": "Session Length", + "shape": "string" + }, + { + "name": "sessionLengthOrder", + "displayName": "Session Length Order", + "shape": "number", + "visible": false + }, + { + "name": "median_session_cost_usd", + "displayName": "Median Session Cost", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 4 + } + ], + "role": "value" + }, + { + "name": "asOf", + "displayName": "As Of", + "shape": "date", + "visible": false + }, + { + "name": "citation", + "displayName": "Citation", + "shape": "string", + "visible": false + } + ], + "options": { + "minimumRequestIntervalSeconds": 86400 + }, + "timeframes": false, + "ui": [ + { + "type": "autocomplete", + "name": "harness", + "label": "Harness", + "isClearable": true, + "allowCustomValues": true, + "help": "Select a coding-agent harness, or enter the slug of another harness OpenRouter publishes", + "data": { + "source": "fixed", + "values": [ + { + "value": "claude-code", + "label": "Claude Code" + }, + { + "value": "codex", + "label": "Codex" + }, + { + "value": "hermes-agent", + "label": "Hermes Agent" + }, + { + "value": "kilo-code", + "label": "Kilo Code" + } + ] + } + }, + { + "type": "choiceChips", + "name": "turnRange", + "label": "Session length", + "options": [ + { + "value": "1-turn", + "label": "1 turn" + }, + { + "value": "2-9-turns", + "label": "2\u20139 turns" + }, + { + "value": "10-49-turns", + "label": "10\u201349 turns" + }, + { + "value": "50-plus-turns", + "label": "50+ turns" + } + ] + }, + { + "type": "objects", + "name": "models", + "label": "Model(s)", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Model" + ] + } + } + } + ] +} diff --git a/plugins/OpenRouter/v1/dataStreams/taskClassifications.json b/plugins/OpenRouter/v1/dataStreams/taskClassifications.json index 2600cc56..e992d717 100644 --- a/plugins/OpenRouter/v1/dataStreams/taskClassifications.json +++ b/plugins/OpenRouter/v1/dataStreams/taskClassifications.json @@ -75,7 +75,14 @@ { "name": "asOf", "displayName": "As Of", - "shape": "date" + "shape": "date", + "visible": false + }, + { + "name": "citation", + "displayName": "Citation", + "shape": "string", + "visible": false } ], "options": { diff --git a/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json b/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json new file mode 100644 index 00000000..f68f62b2 --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json @@ -0,0 +1,429 @@ +{ + "name": "OpenRouter Ecosystem", + "schemaVersion": "1.5", + "timeframe": "last30days", + "dashboard": { + "_type": "layout/grid", + "columns": 6, + "version": 1, + "contents": [ + { + "i": "adfbf427-f14a-479a-ba73-0992167fb17c", + "x": 0, + "y": 0, + "w": 6, + "h": 5, + "config": { + "_type": "tile/data-stream", + "title": "Trending Apps On OpenRouter", + "description": "Up to 25 public apps whose tokens grew the most on OpenRouter compared with the three previous periods of the same length, sized by tokens.", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[appRankings]}}", + "name": "appRankings", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "rankBy": "trending" + }, + "sort": { + "by": [ + [ + "rank", + "asc" + ] + ], + "top": 25 + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "treemap", + "encoding": { + "label": { + "field": "app_name", + "type": "nominal", + "title": "App" + }, + "value": { + "field": "total_tokens", + "type": "quantitative", + "title": "Tokens" + } + }, + "style": { + "legendPosition": "none" + } + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "app_name", + "yAxisData": [ + "total_tokens" + ], + "xAxisGroup": "none", + "xAxisLabel": "App", + "yAxisLabel": "Tokens", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "c0e54f6c-51f5-4708-be03-4b9b4fdb1c2a", + "x": 0, + "y": 5, + "w": 3, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Top Apps On OpenRouter", + "description": "Public apps with the most tokens on OpenRouter.", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[appRankings]}}", + "name": "appRankings", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "rankBy": "popular" + }, + "sort": { + "by": [ + [ + "rank", + "asc" + ] + ], + "top": 10 + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "app_name", + "yAxisData": [ + "total_tokens" + ], + "xAxisGroup": "none", + "xAxisLabel": "App", + "yAxisLabel": "Tokens", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "51dfb910-bec4-45ea-b9a2-683c1574f00f", + "x": 0, + "y": 9, + "w": 4, + "h": 5, + "config": { + "_type": "tile/data-stream", + "title": "Tasks On OpenRouter", + "description": "By Share Of Requests", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[taskClassifications]}}", + "name": "taskClassifications", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "usageShare", + "desc" + ] + ] + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "treemap", + "encoding": { + "parent": { + "field": "category", + "type": "nominal", + "title": "Category" + }, + "label": { + "field": "task", + "type": "nominal", + "title": "Task" + }, + "value": { + "field": "usageShare", + "type": "quantitative", + "title": "Request Share" + } + } + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "task", + "yAxisData": [ + "usageShare" + ], + "xAxisGroup": "none", + "xAxisLabel": "Task", + "yAxisLabel": "Request Share", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "0472205a-cff6-442b-93ec-2b063b6aa157", + "x": 4, + "y": 9, + "w": 2, + "h": 5, + "config": { + "_type": "tile/data-stream", + "title": "Request Share By Task Category", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[taskClassifications]}}", + "name": "taskClassifications", + "pluginConfigId": "{{configId}}", + "group": { + "by": [ + [ + "category", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "usageShare" + ] + } + ] + }, + "sort": { + "by": [ + [ + "usageShare_sum", + "desc" + ] + ] + }, + "metadata": [ + { + "name": "usageShare_sum", + "shape": [ + "percent", + { + "asZeroToOne": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "pie", + "encoding": { + "label": { + "field": "category_uniqueValues", + "type": "nominal", + "title": "Category" + }, + "value": { + "field": "usageShare_sum", + "type": "quantitative", + "title": "Request Share" + } + }, + "style": { + "pieInnerRadius": 55, + "legendPosition": "bottom" + } + } + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "usageShare_sum", + "labelColumn": "category_uniqueValues", + "hideCenterValue": false, + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "i": "488c8839-874c-4a1e-a32f-5318792e4b68", + "x": 3, + "y": 5, + "w": 3, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Providers By Headquarters", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[providers]}}", + "name": "providers", + "pluginConfigId": "{{configId}}", + "group": { + "by": [ + [ + "headquarters", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "count" + } + ] + }, + "sort": { + "by": [ + [ + "count", + "desc" + ] + ] + }, + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "headquarters", + "operation": "notempty" + } + ] + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "pie", + "encoding": { + "label": { + "field": "headquarters_uniqueValues", + "type": "nominal", + "title": "Headquarters" + }, + "value": { + "field": "count", + "type": "quantitative", + "title": "Providers" + } + }, + "style": { + "legendPosition": "bottom" + } + } + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "count", + "labelColumn": "headquarters_uniqueValues", + "hideCenterValue": false, + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "i": "84870314-2f06-4a46-af2b-ca3f40bfa2f2", + "x": 0, + "y": 14, + "w": 6, + "h": 1, + "config": { + "_type": "tile/text", + "title": "", + "description": "", + "visualisation": { + "config": { + "content": "Apps: OpenRouter ([openrouter.ai/apps](https://openrouter.ai/apps)). Tasks: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)), covering the 7 days to yesterday (UTC). \nLicensed under CC BY 4.0.", + "fontSize": 12, + "align": "left", + "autoSize": true + } + } + } + } + ] + } +} diff --git a/plugins/OpenRouter/v1/defaultContent/manifest.json b/plugins/OpenRouter/v1/defaultContent/manifest.json index 12683324..cc0d6c63 100644 --- a/plugins/OpenRouter/v1/defaultContent/manifest.json +++ b/plugins/OpenRouter/v1/defaultContent/manifest.json @@ -8,6 +8,10 @@ "name": "modelComparison", "type": "dashboard" }, + { + "name": "ecosystem", + "type": "dashboard" + }, { "name": "model", "type": "dashboard" diff --git a/plugins/OpenRouter/v1/defaultContent/model.dash.json b/plugins/OpenRouter/v1/defaultContent/model.dash.json index e38ab5a1..fee1b80d 100644 --- a/plugins/OpenRouter/v1/defaultContent/model.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/model.dash.json @@ -653,9 +653,102 @@ } }, { - "i": "c4ac55e8-5333-45eb-ace6-c5e7aad15b6f", + "i": "270958b4-1c9b-4063-8f93-f9908be878c4", "x": 0, "y": 12, + "w": 4, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Session Cost By Harness", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[sessionCost]}}", + "name": "sessionCost", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "models": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + } + }, + "sort": { + "by": [ + [ + "sessionLengthOrder", + "asc" + ], + [ + "app_name", + "asc" + ] + ] + } + }, + "variables": [ + "{{variables.[Model]}}" + ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "app_name", + "type": "nominal", + "title": "Harness" + }, + "y": { + "field": "median_session_cost_usd", + "type": "quantitative", + "title": "Median Session Cost ($)" + }, + "series": { + "field": "sessionLength", + "type": "nominal", + "title": "Session Length" + } + }, + "style": { + "legendPosition": "bottom", + "cartesianStack": "off", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Median Session Cost ($)" + } + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "app_name", + "sessionLength", + "median_session_cost_usd", + "model_permaslug" + ], + "hiddenColumns": [ + "app_slug", + "turn_range", + "sessionLengthOrder", + "asOf", + "citation" + ] + } + } + } + } + }, + { + "i": "c4ac55e8-5333-45eb-ace6-c5e7aad15b6f", + "x": 0, + "y": 16, "w": 2, "h": 3, "moved": false, @@ -723,7 +816,7 @@ { "i": "e81a0ea0-b5fd-4021-8737-2c50997ab7a4", "x": 2, - "y": 12, + "y": 16, "w": 2, "h": 3, "moved": false, @@ -790,7 +883,7 @@ { "i": "041f6c43-1bf6-4e38-80bb-ca48e03ab8f0", "x": 0, - "y": 15, + "y": 19, "w": 4, "h": 7, "config": { @@ -895,6 +988,26 @@ } } } + }, + { + "i": "c5b9dd64-17b8-428e-b527-1469545543f8", + "x": 0, + "y": 26, + "w": 4, + "h": 1, + "config": { + "_type": "tile/text", + "title": "", + "description": "", + "visualisation": { + "config": { + "content": "Benchmarks: Artificial Analysis ([artificialanalysis.ai](https://artificialanalysis.ai)), Design Arena ([www.designarena.ai](https://www.designarena.ai)), and OpenRouter evals ([openrouter.ai](https://openrouter.ai)), via OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). \nSession cost: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)), refreshed weekly. \nLicensed under CC BY 4.0.", + "fontSize": 12, + "align": "left", + "autoSize": true + } + } + } } ] } diff --git a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json index 27c45fbd..f7f1dd57 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json @@ -483,7 +483,7 @@ "config": { "_type": "tile/data-stream", "title": "Top Models On OpenRouter By Week (This Year)", - "description": "", + "description": "Each week's 20 most used models across OpenRouter, by tokens. All other models in that week are combined into Other.", "timeframe": "thisYear", "activePluginConfigIds": [ "{{configId}}" @@ -494,7 +494,7 @@ "pluginConfigId": "{{configId}}", "dataSourceConfig": { "period": "week", - "top": 10 + "top": 20 }, "sort": { "by": [ @@ -552,184 +552,11 @@ } } }, - { - "i": "51dfb910-bec4-45ea-b9a2-683c1574f00f", - "x": 0, - "y": 14, - "w": 4, - "h": 5, - "config": { - "_type": "tile/data-stream", - "title": "Tasks On OpenRouter", - "description": "By Share Of Requests", - "timeframe": "none", - "activePluginConfigIds": [ - "{{configId}}" - ], - "dataStream": { - "id": "{{dataStreams.[taskClassifications]}}", - "name": "taskClassifications", - "pluginConfigId": "{{configId}}", - "sort": { - "by": [ - [ - "usageShare", - "desc" - ] - ] - } - }, - "vizSpec": { - "version": "1.0", - "visualization": { - "chartType": "treemap", - "encoding": { - "parent": { - "field": "category", - "type": "nominal", - "title": "Category" - }, - "label": { - "field": "task", - "type": "nominal", - "title": "Task" - }, - "value": { - "field": "usageShare", - "type": "quantitative", - "title": "Request Share" - } - } - } - }, - "visualisation": { - "type": "data-stream-bar-chart", - "config": { - "data-stream-bar-chart": { - "xAxisData": "task", - "yAxisData": [ - "usageShare" - ], - "xAxisGroup": "none", - "xAxisLabel": "Task", - "yAxisLabel": "Request Share", - "showXAxisLabel": true, - "showYAxisLabel": true, - "showLegend": false, - "legendPosition": "bottom", - "showGrid": true, - "horizontalLayout": "horizontal", - "displayMode": "actual", - "showTotals": false, - "showValue": false, - "grouping": false, - "range": { - "type": "auto" - } - } - } - } - } - }, - { - "i": "0472205a-cff6-442b-93ec-2b063b6aa157", - "x": 4, - "y": 14, - "w": 2, - "h": 5, - "config": { - "_type": "tile/data-stream", - "title": "Request Share By Task Category", - "description": "", - "timeframe": "none", - "activePluginConfigIds": [ - "{{configId}}" - ], - "dataStream": { - "id": "{{dataStreams.[taskClassifications]}}", - "name": "taskClassifications", - "pluginConfigId": "{{configId}}", - "group": { - "by": [ - [ - "category", - "uniqueValues" - ] - ], - "aggregate": [ - { - "type": "sum", - "names": [ - "usageShare" - ] - } - ] - }, - "sort": { - "by": [ - [ - "usageShare_sum", - "desc" - ] - ] - }, - "metadata": [ - { - "name": "usageShare_sum", - "shape": [ - "percent", - { - "asZeroToOne": true - } - ] - }, - { - "pattern": ".*" - } - ] - }, - "vizSpec": { - "version": "1.0", - "visualization": { - "chartType": "pie", - "encoding": { - "label": { - "field": "category_uniqueValues", - "type": "nominal", - "title": "Category" - }, - "value": { - "field": "usageShare_sum", - "type": "quantitative", - "title": "Request Share" - } - }, - "style": { - "pieInnerRadius": 55, - "legendPosition": "bottom" - } - } - }, - "visualisation": { - "type": "data-stream-donut-chart", - "config": { - "data-stream-donut-chart": { - "valueColumn": "usageShare_sum", - "labelColumn": "category_uniqueValues", - "hideCenterValue": false, - "showValuesAsPercentage": true, - "legendPosition": "auto", - "legendMode": "table" - } - } - } - } - }, { "i": "a3384d79-77a8-49c0-bb08-62d6744303e1", "x": 0, - "y": 19, - "w": 4, + "y": 14, + "w": 6, "h": 4, "config": { "_type": "tile/data-stream", @@ -767,7 +594,8 @@ ], "hiddenColumns": [ "tag", - "tokenShare" + "tokenShare", + "citation" ] } } @@ -775,85 +603,89 @@ } }, { - "i": "488c8839-874c-4a1e-a32f-5318792e4b68", - "x": 4, - "y": 19, - "w": 2, - "h": 4, + "i": "37f6bcdf-02a2-4c2a-861d-c9515d1c2092", + "x": 0, + "y": 18, + "w": 6, + "h": 8, "config": { "_type": "tile/data-stream", - "title": "Providers By Headquarters", - "description": "", + "title": "Claude Code Cost Per Session", + "description": "Median cost of one Claude Code session routed through OpenRouter, by model and session length. Users can pick any model on OpenRouter, so many aren't Anthropic's.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[providers]}}", - "name": "providers", + "id": "{{dataStreams.[sessionCost]}}", + "name": "sessionCost", "pluginConfigId": "{{configId}}", - "group": { - "by": [ - [ - "headquarters", - "uniqueValues" - ] - ], - "aggregate": [ - { - "type": "count" - } - ] + "dataSourceConfig": { + "harness": "claude-code" }, "sort": { "by": [ [ - "count", - "desc" + "sessionLengthOrder", + "asc" + ], + [ + "median_session_cost_usd", + "asc" ] ] - }, - "filter": { - "multiOperation": "and", - "filters": [ - { - "column": "headquarters", - "operation": "notempty" - } - ] } }, "vizSpec": { "version": "1.0", "visualization": { - "chartType": "pie", + "chartType": "scatter", "encoding": { - "label": { - "field": "headquarters_uniqueValues", + "x": { + "field": "model_permaslug", "type": "nominal", - "title": "Headquarters" + "title": "Model" }, - "value": { - "field": "count", + "y": { + "field": "median_session_cost_usd", "type": "quantitative", - "title": "Providers" + "title": "Median Session Cost ($)" + }, + "color": { + "field": "sessionLength", + "type": "nominal", + "title": "Session Length" } }, "style": { - "legendPosition": "bottom" + "legendPosition": "bottom", + "cartesianYAxisScale": "log", + "cartesianXAxisLabelRotation": 45, + "cartesianXAxisTickDensity": "dense", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Median Session Cost ($, log scale)", + "cartesianShowVerticalGridLines": true } } }, "visualisation": { - "type": "data-stream-donut-chart", + "type": "data-stream-table", "config": { - "data-stream-donut-chart": { - "valueColumn": "count", - "labelColumn": "headquarters_uniqueValues", - "hideCenterValue": false, - "showValuesAsPercentage": true, - "legendPosition": "auto", - "legendMode": "table" + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "model_permaslug", + "sessionLength", + "median_session_cost_usd", + "app_name" + ], + "hiddenColumns": [ + "app_slug", + "turn_range", + "sessionLengthOrder", + "asOf", + "citation" + ] } } } @@ -862,7 +694,7 @@ { "i": "56d4d9f8-9cc4-4852-9306-2997d9e87000", "x": 0, - "y": 23, + "y": 26, "w": 6, "h": 8, "config": { @@ -1014,7 +846,7 @@ { "i": "679f0166-77c4-47c1-9ae5-7e4c1721f718", "x": 0, - "y": 31, + "y": 34, "w": 6, "h": 1, "config": { @@ -1023,7 +855,7 @@ "description": "", "visualisation": { "config": { - "content": "Benchmark data from Artificial Analysis, Design Arena, and OpenRouter, via OpenRouter. Task data is as of the **As Of** date shown in the Top Model By Task table. \nSource: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)), licensed CC BY 4.0", + "content": "Benchmarks: Artificial Analysis ([artificialanalysis.ai](https://artificialanalysis.ai)), Design Arena ([www.designarena.ai](https://www.designarena.ai)), and OpenRouter evals ([openrouter.ai](https://openrouter.ai)), via OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). \nRankings, tasks, and session cost: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). Task data is as of the **As Of** date in the Top Model By Task table; session cost is refreshed weekly. \nLicensed under CC BY 4.0.", "fontSize": 12, "align": "left", "autoSize": true diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json index 9f3136ab..8fe5ccce 100644 --- a/plugins/OpenRouter/v1/defaultContent/overview.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -715,9 +715,9 @@ "z": 0, "config": { "_type": "tile/data-stream", - "title": "Latency p50 By Model (Last 7 Days)", + "title": "Latency p50 By Model (Last 30 Days)", "description": "", - "timeframe": "last7days", + "timeframe": "last30days", "activePluginConfigIds": [ "{{configId}}" ], diff --git a/plugins/OpenRouter/v1/docs/README.md b/plugins/OpenRouter/v1/docs/README.md index 8957634e..70f8ffb8 100644 --- a/plugins/OpenRouter/v1/docs/README.md +++ b/plugins/OpenRouter/v1/docs/README.md @@ -32,9 +32,9 @@ On save, the plugin checks the key in two steps. An invalid, disabled, or expire - **Model comparison** — pricing, context length, modalities, and Artificial Analysis benchmark scores for every model on OpenRouter. - **Providers** — every provider serving models on OpenRouter, with headquarters and datacenter locations. - **Provider performance** — current status, price, uptime, latency, and throughput for each provider serving a model. -- **Benchmarks and rankings** — scores from Artificial Analysis, Design Arena, and OpenRouter's own evals, OpenRouter's public leaderboard of the most used models, and what share of OpenRouter traffic goes to each kind of task. +- **Benchmarks and rankings** — scores from Artificial Analysis, Design Arena, and OpenRouter's own evals, OpenRouter's public leaderboard of the most used models, what share of OpenRouter traffic goes to each kind of task, what a coding-agent session typically costs, and which public apps use OpenRouter most. -The out-of-the-box dashboards include an account **Overview**, a **Model Comparison** dashboard (which works with a standard API key), and a perspective for each **Model**, **Provider**, **Workspace**, and **API Key** (spend tiles need a management key). The Model perspective includes a current view of each provider's status, price, and throughput. +The out-of-the-box dashboards include an account **Overview**; **Model Comparison** and **OpenRouter Ecosystem** dashboards covering models, apps, task types, and providers across all of OpenRouter, which work with a standard API key; and a perspective for each **Model**, **Provider**, **Workspace**, and **API Key** (spend tiles need a management key). The Model perspective includes a current view of each provider's status, price, and throughput. ## Data streams @@ -43,10 +43,12 @@ The out-of-the-box dashboards include an account **Overview**, a **Model Compari - **Workspaces** — workspaces in the account. Account-wide. Management key only. - **Credits** — credits purchased, used, and remaining. Account-wide. - **Models** — every model on OpenRouter with pricing ($ per 1M tokens), context length, modalities, and Artificial Analysis index scores, optionally filtered to selected models. -- **Providers** — every provider on OpenRouter, with headquarters, datacenter countries, and status, privacy, and terms links. +- **Providers** — every provider on OpenRouter, with headquarters and datacenter countries (by name), and status, privacy, and terms links. - **Model Providers** — providers serving a model, with status, price (including cache reads), uptime over the last 5 minutes, 30 minutes, and day, and latency (p50 to p99) and throughput over the last 30 minutes. Per model. - **Benchmarks** — model benchmark scores from Artificial Analysis, Design Arena, and OpenRouter evals, one row per model and benchmark. -- **Model Rankings** — the most used models across all OpenRouter traffic, by tokens processed per day, week, or month. Optionally keep only the top models and combine the rest into **Other**. +- **Model Rankings** — the most used models across all OpenRouter traffic, by tokens processed per day, week, or month, optionally narrowed to a use case, language type, modality, or context length. Optionally keep only each period's top models and combine the rest into **Other**, as OpenRouter's own rankings chart does. +- **App Rankings** — the top 100 public apps on OpenRouter by tokens, ranked by most tokens or by trending growth, optionally filtered to a category or subcategory (such as CLI agents or roleplay). +- **Session Cost** — median cost of one coding-agent session routed through OpenRouter, by harness (such as Claude Code or Codex), model, and session length (1 turn to 50+ turns), optionally filtered to a harness, session length, or selected models. Refreshed weekly by OpenRouter. - **Task Classifications** — share of all OpenRouter requests and tokens by task type (such as code generation or summarization) over the last 7 days, grouped into Code, Data, Agent, and General, with each task's most used model. ## What gets indexed @@ -64,13 +66,13 @@ With a standard API key, only models and providers are imported; the workspace a ## Known limitations -- **Benchmarks, rankings, and task classifications refresh at most once a day** — OpenRouter limits these endpoints to 500 requests a day per account, so the plugin caches them for 24 hours. -- **Rankings licence** — rankings and task classification data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). The **Model Comparison** dashboard credits it already; keep "Source: OpenRouter (openrouter.ai/rankings)" on any dashboard you build from it, adding "as of" and the **As Of** date for task classifications. +- **Benchmarks, rankings, app rankings, task classifications, and session costs refresh at most once a day** — OpenRouter limits these endpoints to 500 requests a day per account, so the plugin caches them for 24 hours. +- **Rankings licence** — rankings, app ranking, task classification, session cost, and benchmark data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Each row of these streams has a hidden **Citation** column with the exact attribution OpenRouter requires, and a hidden **As Of** date. The **Model Comparison**, **OpenRouter Ecosystem**, and **Model** dashboards credit the data already; show the citation on any dashboard you build from it. - **Not every model has benchmark scores** — Artificial Analysis scores cover fewer than half of models; the rest show blank. - **Usage history** can be queried up to a year back. Latency, time to first token, throughput, and grouping by **Provider**, **Origin**, **Country**, **Finish reason**, or **Streamed** are limited to 31 days; longer timeframes return an error. - **Usage timeframes start at 12 hours** — data is bucketed by hour, day, week, or month to suit the range; periods with no activity are left out rather than shown as zero. - **Large usage queries may be truncated** — the stream warns when this happens; narrow the timeframe or filter to specific objects. -- **Model rankings start at 7 days** — the data is daily. +- **Model rankings start at 7 days** — the data is daily. Use case and language segments are weekly estimates from sampled traffic, so they're shown weekly even when **Day** is selected. - **Task classifications cover the last 7 days only** — they come from a sample of OpenRouter traffic, so only shares are available, not request or token counts. - **Provider status is inferred** — OpenRouter doesn't document its endpoint status codes. The plugin shows `0` as healthy, `-1` to `-3` as warning, and anything lower as error, which matches the uptime OpenRouter reports for those endpoints; check **Uptime (5m)** alongside it. - **Provider performance is a live snapshot** — it covers the last 5 minutes to 1 day, with no history beyond what SquaredUp records. From b1415b5fd4f67fb93d82654d0f42f14499bd286e Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Mon, 5 Oct 2026 07:52:44 +0100 Subject: [PATCH 04/16] Add OpenRouter workspace settings and a Model Usage table Squashed from: - Show OpenRouter workspace settings - Add an OpenRouter model usage table with drilldown to the overview - Filter OpenRouter model usage to the selected models - Show OpenRouter spend and cost per 1M tokens by provider on the model dashboard - Address OpenRouter review feedback - Join OpenRouter model usage to models in a SQL tile on the overview Co-Authored-By: Claude Opus 5.5 --- .../OpenRouter/v1/dataStreams/modelUsage.json | 82 ++++++++ plugins/OpenRouter/v1/dataStreams/models.json | 21 ++- .../OpenRouter/v1/dataStreams/rankings.json | 7 +- .../v1/dataStreams/scripts/appRankings.js | 5 +- .../v1/dataStreams/scripts/modelUsage.js | 27 +++ .../v1/dataStreams/scripts/rankings.js | 6 +- .../v1/dataStreams/scripts/usage.js | 10 +- .../v1/dataStreams/scripts/workspaces.js | 11 ++ plugins/OpenRouter/v1/dataStreams/usage.json | 13 +- .../OpenRouter/v1/dataStreams/workspaces.json | 86 ++++++++- .../v1/defaultContent/ecosystem.dash.json | 4 +- .../v1/defaultContent/model.dash.json | 176 ++++++++++++++++-- .../v1/defaultContent/overview.dash.json | 140 +++++++++++++- .../v1/defaultContent/provider.dash.json | 10 +- .../v1/defaultContent/workspace.dash.json | 45 +++-- plugins/OpenRouter/v1/docs/README.md | 12 +- plugins/OpenRouter/v1/ui.json | 2 +- 17 files changed, 592 insertions(+), 65 deletions(-) create mode 100644 plugins/OpenRouter/v1/dataStreams/modelUsage.json create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/modelUsage.js create mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/workspaces.js diff --git a/plugins/OpenRouter/v1/dataStreams/modelUsage.json b/plugins/OpenRouter/v1/dataStreams/modelUsage.json new file mode 100644 index 00000000..3ecac5a7 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/modelUsage.json @@ -0,0 +1,82 @@ +{ + "name": "modelUsage", + "displayName": "Model Usage", + "description": "Spend, requests, cost per request, and tokens for each model over the timeframe, by canonical slug (requires a management key)", + "tags": [ + "Usage", + "Cost" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "analytics/query", + "paging": { + "mode": "none" + }, + "getArgs": [], + "headers": [], + "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; return JSON.stringify({ metrics: ['total_usage', 'request_count', 'tokens_prompt', 'tokens_completion'], dimensions: ['model'], granularity: 'month', time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", + "postRequestScript": "modelUsage.js" + }, + "matches": "none", + "metadata": [ + { + "name": "slug", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "spend", + "displayName": "Spend", + "shape": "usd", + "role": "value" + }, + { + "name": "requests", + "displayName": "Requests", + "shape": "number" + }, + { + "name": "costPerRequest", + "displayName": "Cost Per Request", + "shape": [ + "usd", + { + "decimalPlaces": 6 + } + ] + }, + { + "name": "promptTokens", + "displayName": "Prompt Tokens", + "shape": "number" + }, + { + "name": "completionTokens", + "displayName": "Completion Tokens", + "shape": "number" + } + ], + "timeframes": [ + "last12hours", + "last24hours", + "last7days", + "last30days", + "thisMonth", + "lastMonth", + "thisQuarter", + "lastQuarter", + "thisYear" + ], + "defaultShaping": { + "sort": { + "by": [ + [ + "spend", + "desc" + ] + ] + } + } +} diff --git a/plugins/OpenRouter/v1/dataStreams/models.json b/plugins/OpenRouter/v1/dataStreams/models.json index 7095a89e..8cdec806 100644 --- a/plugins/OpenRouter/v1/dataStreams/models.json +++ b/plugins/OpenRouter/v1/dataStreams/models.json @@ -12,7 +12,12 @@ "paging": { "mode": "none" }, - "getArgs": [], + "getArgs": [ + { + "key": "output_modalities", + "value": "all" + } + ], "headers": [], "expandInnerObjects": true, "postRequestScript": "models.js" @@ -123,7 +128,12 @@ { "name": "knowledge_cutoff", "displayName": "Knowledge Cutoff", - "shape": "string" + "shape": [ + "date", + { + "timeZone": "Etc/UTC" + } + ] }, { "name": "created", @@ -138,7 +148,12 @@ { "name": "expiration_date", "displayName": "Expiration Date", - "shape": "string" + "shape": [ + "date", + { + "timeZone": "Etc/UTC" + } + ] }, { "name": "pricing.prompt", diff --git a/plugins/OpenRouter/v1/dataStreams/rankings.json b/plugins/OpenRouter/v1/dataStreams/rankings.json index 20cd309a..7791d527 100644 --- a/plugins/OpenRouter/v1/dataStreams/rankings.json +++ b/plugins/OpenRouter/v1/dataStreams/rankings.json @@ -206,7 +206,12 @@ { "name": "date", "displayName": "Date", - "shape": "date", + "shape": [ + "date", + { + "timeZone": "Etc/UTC" + } + ], "role": "timestamp" }, { diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/appRankings.js b/plugins/OpenRouter/v1/dataStreams/scripts/appRankings.js index b50dfcca..b64d3380 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/appRankings.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/appRankings.js @@ -1,5 +1,6 @@ -// Adds the as-of date and citation OpenRouter requires to every row +// Adds the as-of date and citation OpenRouter requires to every row, and +// converts total_tokens (sent as a decimal string) to a number const asOf = data?.meta?.as_of; const citation = `Source: OpenRouter (openrouter.ai/apps), as of ${asOf}. Licensed under CC BY 4.0.`; -result = (data?.data ?? []).map((r) => ({ ...r, asOf, citation })); +result = (data?.data ?? []).map((r) => ({ ...r, total_tokens: Number(r.total_tokens), asOf, citation })); diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/modelUsage.js b/plugins/OpenRouter/v1/dataStreams/scripts/modelUsage.js new file mode 100644 index 00000000..b90ec13e --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/modelUsage.js @@ -0,0 +1,27 @@ +// Sums analytics rows per model. Analytics reports models by canonical slug +// (e.g. z-ai/glm-5.3-flash-20260826), not by Model object id; dashboards join +// on the Model objects' canonical_slug property to resolve names and drilldowns. +const body = data?.data ?? {}; + +if (body.metadata?.truncated) { + api.report.warning('Results were truncated by OpenRouter; narrow the timeframe'); +} + +const totals = new Map(); +for (const row of body.data ?? []) { + const slug = row.model; + if (!slug) continue; + + const total = totals.get(slug) ?? { spend: 0, requests: 0, promptTokens: 0, completionTokens: 0 }; + total.spend += Number(row.total_usage ?? 0); + total.requests += Number(row.request_count ?? 0); + total.promptTokens += Number(row.tokens_prompt ?? 0); + total.completionTokens += Number(row.tokens_completion ?? 0); + totals.set(slug, total); +} + +result = [...totals].map(([slug, total]) => ({ + slug, + ...total, + costPerRequest: total.requests ? total.spend / total.requests : null, +})); diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js b/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js index 64d96159..09160bfe 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/rankings.js @@ -3,7 +3,9 @@ // into one "Other" row, matching how openrouter.ai/rankings charts its top // models. Without it, rows pass through. Every row then gets the as-of date // and citation OpenRouter requires. -const rows = data?.data ?? []; + +// total_tokens arrives as a decimal string +const rows = (data?.data ?? []).map((r) => ({ ...r, total_tokens: Number(r.total_tokens) })); const top = Number(context.config.top); if (!(top > 0)) { @@ -13,7 +15,7 @@ if (!(top > 0)) { for (const r of rows) { const period = byDate[r.date] ?? (byDate[r.date] = []); - period.push({ ...r, total_tokens: Number(r.total_tokens) }); + period.push(r); } result = []; diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/usage.js b/plugins/OpenRouter/v1/dataStreams/scripts/usage.js index 6bd542f3..7e976425 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/usage.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/usage.js @@ -1,6 +1,7 @@ // Normalises analytics rows: the time key is suffixed with the granularity // (date__day, or created_at__day for raw-view fields like latency and provider) -// and large counts arrive as strings. +// and large counts arrive as strings. cost_per_million_tokens isn't an OpenRouter +// metric; it's derived from total_usage and tokens_total, requested together. const body = data?.data ?? {}; const metric = context.config.metric || 'total_usage'; const groupBy = context.config.groupBy; @@ -11,6 +12,11 @@ if (body.metadata?.truncated) { ); } +const costPerMillionTokens = (row) => { + const tokens = Number(row.tokens_total ?? 0); + return tokens && row.total_usage != null ? (Number(row.total_usage) / tokens) * 1e6 : null; +}; + const dateKey = (row) => Object.keys(row).find((k) => /__(minute|hour|day|week|month)$/.test(k)); result = (body.data ?? []) @@ -18,7 +24,7 @@ result = (body.data ?? []) const date = row[dateKey(row)]; if (!date) return null; - const value = row[metric]; + const value = metric === 'cost_per_million_tokens' ? costPerMillionTokens(row) : row[metric]; return { date, diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/workspaces.js b/plugins/OpenRouter/v1/dataStreams/scripts/workspaces.js new file mode 100644 index 00000000..b6b83ce5 --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/scripts/workspaces.js @@ -0,0 +1,11 @@ +// Optional `workspaces` object-picker parameter: selected objects arrive at +// context.config.workspaces, each rawId a single-element array. Empty/absent +// (including the import run) returns every workspace. Also flattens the +// disabled server tools list into a comma-separated string for tables. +const unwrap = (v) => (Array.isArray(v) ? v[0] : v); +const selected = context.config?.workspaces ?? []; +const ids = new Set(selected.map((o) => unwrap(o.rawId)).filter(Boolean)); + +result = (data?.data ?? []) + .filter((w) => !ids.size || ids.has(w.id)) + .map((w) => ({ ...w, disabled_server_tools: w.disabled_server_tools?.join(', ') || null })); diff --git a/plugins/OpenRouter/v1/dataStreams/usage.json b/plugins/OpenRouter/v1/dataStreams/usage.json index 652e0e82..1807112c 100644 --- a/plugins/OpenRouter/v1/dataStreams/usage.json +++ b/plugins/OpenRouter/v1/dataStreams/usage.json @@ -15,7 +15,7 @@ }, "getArgs": [], "headers": [], - "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; const days = (new Date(timeframe.end) - new Date(timeframe.start)) / 864e5; const granularity = days <= 2 ? 'hour' : days <= 31 ? 'day' : days <= 92 ? 'week' : 'month'; const selected = typeof filterObjects !== 'undefined' && Array.isArray(filterObjects) ? filterObjects : []; const one = (v) => (Array.isArray(v) ? v[0] : v); const idsOf = (type, ...props) => [...new Set(selected.filter((o) => one(o.sourceType) === type).flatMap((o) => props.map((p) => one(o[p]))).filter(Boolean))]; const rules = [ ['model', idsOf('Model', 'rawId', 'canonical_slug')], ['provider', idsOf('Provider', 'name')], ['workspace', idsOf('Workspace', 'rawId')], ['api_key_id', idsOf('API Key', 'rawId')] ]; const filters = rules.filter(([, values]) => values.length).map(([field, value]) => ({ field, operator: 'in', value })); return JSON.stringify({ metrics: [typeof metric !== 'undefined' && metric ? metric : 'total_usage'], dimensions: typeof groupBy !== 'undefined' && groupBy ? [groupBy] : [], filters, granularity, time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", + "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; const days = (new Date(timeframe.end) - new Date(timeframe.start)) / 864e5; const granularity = days <= 2 ? 'hour' : days <= 31 ? 'day' : days <= 92 ? 'week' : 'month'; const selected = typeof filterObjects !== 'undefined' && Array.isArray(filterObjects) ? filterObjects : []; const one = (v) => (Array.isArray(v) ? v[0] : v); const idsOf = (type, ...props) => [...new Set(selected.filter((o) => one(o.sourceType) === type).flatMap((o) => props.map((p) => one(o[p]))).filter(Boolean))]; const rules = [ ['model', idsOf('Model', 'rawId', 'canonical_slug')], ['provider', idsOf('Provider', 'name')], ['workspace', idsOf('Workspace', 'rawId')], ['api_key_id', idsOf('API Key', 'rawId')] ]; const filters = rules.filter(([, values]) => values.length).map(([field, value]) => ({ field, operator: 'in', value })); const selectedMetric = typeof metric !== 'undefined' && metric ? metric : 'total_usage'; return JSON.stringify({ metrics: selectedMetric === 'cost_per_million_tokens' ? ['total_usage', 'tokens_total'] : [selectedMetric], dimensions: typeof groupBy !== 'undefined' && groupBy ? [groupBy] : [], filters, granularity, time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", "postRequestScript": "usage.js" }, "matches": "none", @@ -44,6 +44,10 @@ { "value": "byok_usage", "label": "BYOK spend (USD)" + }, + { + "value": "cost_per_million_tokens", + "label": "Cost per 1M tokens (USD)" } ] }, @@ -193,7 +197,12 @@ { "name": "date", "displayName": "Date", - "shape": "date", + "shape": [ + "date", + { + "timeZone": "Etc/UTC" + } + ], "role": "timestamp" }, { diff --git a/plugins/OpenRouter/v1/dataStreams/workspaces.json b/plugins/OpenRouter/v1/dataStreams/workspaces.json index b0813319..ebb79875 100644 --- a/plugins/OpenRouter/v1/dataStreams/workspaces.json +++ b/plugins/OpenRouter/v1/dataStreams/workspaces.json @@ -1,7 +1,7 @@ { "name": "workspaces", "displayName": "Workspaces", - "description": "Workspaces in the OpenRouter account (requires a management key)", + "description": "Workspaces in the OpenRouter account, with default models, logging, and budget settings (requires a management key)", "tags": ["Workspaces"], "baseDataSourceName": "httpRequestUnscoped", "config": { @@ -19,7 +19,7 @@ }, "getArgs": [], "headers": [], - "pathToData": "data" + "postRequestScript": "workspaces.js" }, "matches": "none", "metadata": [ @@ -38,7 +38,87 @@ "name": "slug", "displayName": "Slug", "shape": "string" + }, + { + "name": "description", + "displayName": "Description", + "shape": "string" + }, + { + "name": "default_text_model", + "displayName": "Default Text Model", + "shape": "string" + }, + { + "name": "default_image_model", + "displayName": "Default Image Model", + "shape": "string" + }, + { + "name": "default_provider_sort", + "displayName": "Default Provider Sort", + "shape": "string" + }, + { + "name": "disabled_server_tools", + "displayName": "Disabled Server Tools", + "shape": "string" + }, + { + "name": "include_byok_in_budgets", + "displayName": "BYOK In Budgets", + "shape": "boolean" + }, + { + "name": "is_observability_io_logging_enabled", + "displayName": "Private Logging", + "shape": "boolean" + }, + { + "name": "io_logging_sampling_rate", + "displayName": "Private Logging Sample Rate", + "shape": [ + "percent", + { + "asZeroToOne": true + } + ] + }, + { + "name": "is_observability_broadcast_enabled", + "displayName": "Broadcast", + "shape": "boolean" + }, + { + "name": "is_data_discount_logging_enabled", + "displayName": "Data Discount Logging", + "shape": "boolean" + }, + { + "name": "created_at", + "displayName": "Created", + "shape": "date" + }, + { + "name": "updated_at", + "displayName": "Updated", + "shape": "date" } ], - "timeframes": false + "timeframes": false, + "ui": [ + { + "type": "objects", + "name": "workspaces", + "label": "Workspace(s)", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Workspace" + ] + } + } + } + ] } diff --git a/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json b/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json index f68f62b2..392bb014 100644 --- a/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json @@ -1,5 +1,5 @@ { - "name": "OpenRouter Ecosystem", + "name": "Ecosystem", "schemaVersion": "1.5", "timeframe": "last30days", "dashboard": { @@ -155,7 +155,7 @@ "config": { "_type": "tile/data-stream", "title": "Tasks On OpenRouter", - "description": "By Share Of Requests", + "description": "Task types ranked by their share of requests on OpenRouter.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" diff --git a/plugins/OpenRouter/v1/defaultContent/model.dash.json b/plugins/OpenRouter/v1/defaultContent/model.dash.json index fee1b80d..64a0ccc0 100644 --- a/plugins/OpenRouter/v1/defaultContent/model.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/model.dash.json @@ -444,10 +444,162 @@ } }, { - "i": "0e72d465-f7f3-42ad-8a6f-2b7a91a8850a", + "i": "47caf67c-f374-458b-afd2-7ba9b3cc4979", "x": 0, "y": 7, "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Spend By Provider (Last 30 Days)", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "total_usage", + "groupBy": "provider", + "filterObjects": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "group", + "showLegend": true, + "legendPosition": "bottom", + "yAxisLabel": "Spend ($)", + "showYAxisLabel": true, + "showTrendLine": false + } + } + }, + "timeframe": "last30days" + } + }, + { + "i": "152fb443-9a06-4eea-a5c4-a4dafd4c0277", + "x": 2, + "y": 7, + "w": 2, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Your Cost Per 1M Tokens By Provider (Last 30 Days)", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "metric": "cost_per_million_tokens", + "groupBy": "provider", + "filterObjects": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + } + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 4 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Model]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "group", + "showLegend": true, + "legendPosition": "bottom", + "yAxisLabel": "Cost per 1M tokens ($)", + "showYAxisLabel": true, + "showTrendLine": false + } + } + }, + "timeframe": "last30days" + } + }, + { + "i": "0e72d465-f7f3-42ad-8a6f-2b7a91a8850a", + "x": 0, + "y": 10, + "w": 2, "h": 2, "moved": false, "static": false, @@ -515,7 +667,7 @@ { "i": "0fe94a89-e194-4ce1-bf03-3495bec434cd", "x": 2, - "y": 7, + "y": 10, "w": 2, "h": 2, "moved": false, @@ -584,7 +736,7 @@ { "i": "21bb9221-3105-46cb-a988-b677dafef3e9", "x": 0, - "y": 9, + "y": 12, "w": 4, "h": 3, "moved": false, @@ -655,7 +807,7 @@ { "i": "270958b4-1c9b-4063-8f93-f9908be878c4", "x": 0, - "y": 12, + "y": 15, "w": 4, "h": 4, "config": { @@ -748,7 +900,7 @@ { "i": "c4ac55e8-5333-45eb-ace6-c5e7aad15b6f", "x": 0, - "y": 16, + "y": 19, "w": 2, "h": 3, "moved": false, @@ -793,8 +945,8 @@ "latency_last_30m.p90" ], "xAxisGroup": "none", - "xAxisLabel": "Provider", - "yAxisLabel": "Latency (ms)", + "xAxisLabel": "Endpoint", + "yAxisLabel": "Latency", "showXAxisLabel": true, "showYAxisLabel": true, "showLegend": true, @@ -816,7 +968,7 @@ { "i": "e81a0ea0-b5fd-4021-8737-2c50997ab7a4", "x": 2, - "y": 16, + "y": 19, "w": 2, "h": 3, "moved": false, @@ -860,7 +1012,7 @@ "throughput_last_30m.p50" ], "xAxisGroup": "none", - "xAxisLabel": "Provider", + "xAxisLabel": "Endpoint", "yAxisLabel": "Tokens/s", "showXAxisLabel": true, "showYAxisLabel": true, @@ -883,7 +1035,7 @@ { "i": "041f6c43-1bf6-4e38-80bb-ca48e03ab8f0", "x": 0, - "y": 19, + "y": 22, "w": 4, "h": 7, "config": { @@ -992,7 +1144,7 @@ { "i": "c5b9dd64-17b8-428e-b527-1469545543f8", "x": 0, - "y": 26, + "y": 29, "w": 4, "h": 1, "config": { @@ -1001,7 +1153,7 @@ "description": "", "visualisation": { "config": { - "content": "Benchmarks: Artificial Analysis ([artificialanalysis.ai](https://artificialanalysis.ai)), Design Arena ([www.designarena.ai](https://www.designarena.ai)), and OpenRouter evals ([openrouter.ai](https://openrouter.ai)), via OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). \nSession cost: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)), refreshed weekly. \nLicensed under CC BY 4.0.", + "content": "Benchmarks: Artificial Analysis ([artificialanalysis.ai](https://artificialanalysis.ai)), via OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). \nSession cost: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)), refreshed weekly. \nLicensed under CC BY 4.0.", "fontSize": 12, "align": "left", "autoSize": true diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json index 8fe5ccce..70a25672 100644 --- a/plugins/OpenRouter/v1/defaultContent/overview.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -429,9 +429,139 @@ } }, { - "i": "55c667d9-63af-4556-962d-87a6325df855", + "i": "c6b79b6d-a751-4eaa-ac22-7721b5dfba27", "x": 0, "y": 7, + "w": 4, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Model Usage", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "datastream-sql", + "dataSourceConfig": { + "version": "2.0", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n sourceId AS id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n m.id,\n COALESCE(m.name, u.slug) AS name,\n u.slug,\n u.spend,\n u.requests,\n u.costPerRequest,\n u.promptTokens,\n u.completionTokens\nFROM Usage u\nLEFT JOIN by_slug m ON m.slug = u.slug\nORDER BY u.spend DESC", + "tables": [ + { + "tableName": "Usage", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[modelUsage]}}", + "name": "modelUsage", + "pluginConfigId": "{{configId}}" + } + } + }, + { + "tableName": "Models", + "config": { + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}" + }, + "timeframe": "none" + } + } + ] + }, + "metadata": [ + { + "name": "id", + "displayName": "Model ID", + "shape": "string", + "visible": false + }, + { + "name": "name", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "sourceId": "id", + "sourceType": "Model", + "name": "name" + }, + { + "name": "slug", + "displayName": "Model Slug", + "shape": "string", + "visible": false + }, + { + "name": "spend", + "displayName": "Spend", + "shape": "usd", + "role": "value" + }, + { + "name": "requests", + "displayName": "Requests", + "shape": "number" + }, + { + "name": "costPerRequest", + "displayName": "Cost Per Request", + "shape": [ + "usd", + { + "decimalPlaces": 6 + } + ] + }, + { + "name": "promptTokens", + "displayName": "Prompt Tokens", + "shape": "number" + }, + { + "name": "completionTokens", + "displayName": "Completion Tokens", + "shape": "number" + } + ] + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "name", + "spend", + "requests", + "costPerRequest", + "promptTokens", + "completionTokens" + ], + "hiddenColumns": [ + "id", + "slug" + ] + } + } + } + } + }, + { + "i": "55c667d9-63af-4556-962d-87a6325df855", + "x": 0, + "y": 10, "w": 2, "h": 3, "moved": false, @@ -525,7 +655,7 @@ { "i": "7c94001a-4946-45e0-8d08-08f2b89153ef", "x": 2, - "y": 7, + "y": 10, "w": 2, "h": 3, "moved": false, @@ -605,7 +735,7 @@ { "i": "da913ee7-05b2-4ce2-b11a-64b91f9d1537", "x": 0, - "y": 10, + "y": 13, "w": 2, "h": 3, "moved": false, @@ -707,7 +837,7 @@ { "i": "3bf76eb4-561e-4bae-a971-ba12c6c6d216", "x": 2, - "y": 10, + "y": 13, "w": 2, "h": 3, "moved": false, @@ -795,7 +925,7 @@ { "i": "d596864a-98e3-466c-ba4d-cd59ee6f7f01", "x": 0, - "y": 13, + "y": 16, "w": 4, "h": 3, "moved": false, diff --git a/plugins/OpenRouter/v1/defaultContent/provider.dash.json b/plugins/OpenRouter/v1/defaultContent/provider.dash.json index b27feef3..dcae404b 100644 --- a/plugins/OpenRouter/v1/defaultContent/provider.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/provider.dash.json @@ -251,13 +251,7 @@ "metadata": [ { "name": "value", - "shape": [ - "number", - { - "decimalPlaces": 0, - "thousandsSeparator": true - } - ] + "shape": "milliseconds" }, { "pattern": ".*" @@ -278,7 +272,7 @@ "seriesColumn": "none", "showLegend": false, "legendPosition": "bottom", - "yAxisLabel": "Latency (ms)", + "yAxisLabel": "Latency p50", "showYAxisLabel": true, "showTrendLine": false } diff --git a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json index a21ba7fc..7d5f8aa8 100644 --- a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json @@ -21,17 +21,23 @@ "z": 0, "config": { "_type": "tile/data-stream", - "title": "Properties", + "title": "Settings", "description": "", "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], "dataStream": { - "id": "datastream-properties", - "name": "properties" - }, - "scope": { - "scope": "{{scopes.[Workspaces]}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.[Workspace]}}" + "id": "{{dataStreams.[workspaces]}}", + "name": "workspaces", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "workspaces": { + "scope": "{{scopes.[Workspaces]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Workspace]}}" + } + } }, "variables": [ "{{variables.[Workspace]}}" @@ -43,17 +49,22 @@ "transpose": true, "columnOrder": [ "name", - "slug" + "slug", + "description", + "default_text_model", + "default_image_model", + "default_provider_sort", + "disabled_server_tools", + "include_byok_in_budgets", + "is_observability_io_logging_enabled", + "io_logging_sampling_rate", + "is_observability_broadcast_enabled", + "is_data_discount_logging_enabled", + "created_at", + "updated_at" ], "hiddenColumns": [ - "id", - "links", - "type", - "label", - "sourceId", - "sourceType", - "sourceName", - "rawId" + "id" ] } } diff --git a/plugins/OpenRouter/v1/docs/README.md b/plugins/OpenRouter/v1/docs/README.md index 70f8ffb8..adabfd8b 100644 --- a/plugins/OpenRouter/v1/docs/README.md +++ b/plugins/OpenRouter/v1/docs/README.md @@ -34,13 +34,14 @@ On save, the plugin checks the key in two steps. An invalid, disabled, or expire - **Provider performance** — current status, price, uptime, latency, and throughput for each provider serving a model. - **Benchmarks and rankings** — scores from Artificial Analysis, Design Arena, and OpenRouter's own evals, OpenRouter's public leaderboard of the most used models, what share of OpenRouter traffic goes to each kind of task, what a coding-agent session typically costs, and which public apps use OpenRouter most. -The out-of-the-box dashboards include an account **Overview**; **Model Comparison** and **OpenRouter Ecosystem** dashboards covering models, apps, task types, and providers across all of OpenRouter, which work with a standard API key; and a perspective for each **Model**, **Provider**, **Workspace**, and **API Key** (spend tiles need a management key). The Model perspective includes a current view of each provider's status, price, and throughput. +The out-of-the-box dashboards include an account **Overview**; **Model Comparison** and **Ecosystem** dashboards covering models, apps, task types, and providers across all of OpenRouter, which work with a standard API key; and a perspective for each **Model**, **Provider**, **Workspace**, and **API Key** (spend tiles need a management key). The Model perspective includes a current view of each provider's status, price, and throughput, and your spend and cost per 1M tokens on each provider. ## Data streams -- **Usage** — spend, requests, tokens, cache hit rate, latency, time to first token, or throughput over time, optionally grouped by model, provider, workspace, API key, app, user, origin, country, finish reason, or streamed, and optionally filtered to selected models, providers, workspaces, or API keys. Any other OpenRouter analytics metric or dimension can be typed in. Account-wide. Management key only. +- **Usage** — spend, cost per 1M tokens, requests, tokens, cache hit rate, latency, time to first token, or throughput over time, optionally grouped by model, provider, workspace, API key, app, user, origin, country, finish reason, or streamed, and optionally filtered to selected models, providers, workspaces, or API keys. Any other OpenRouter analytics metric or dimension can be typed in. Account-wide. Management key only. +- **Model Usage** — spend, requests, cost per request, and prompt and completion tokens for every model over the timeframe, keyed by canonical slug. OpenRouter reports `:batch` variants under the same slug as their base model, so their usage is combined. The overview dashboard joins this to the imported models for names and drilldown. Account-wide. Management key only. - **API Keys** — API keys in a workspace with spend and limits. Per workspace. Management key only. -- **Workspaces** — workspaces in the account. Account-wide. Management key only. +- **Workspaces** — workspaces in the account with their default models, provider sort, disabled server tools, whether BYOK spend counts toward budgets, and logging and broadcast settings, optionally filtered to selected workspaces. Account-wide. Management key only. - **Credits** — credits purchased, used, and remaining. Account-wide. - **Models** — every model on OpenRouter with pricing ($ per 1M tokens), context length, modalities, and Artificial Analysis index scores, optionally filtered to selected models. - **Providers** — every provider on OpenRouter, with headquarters and datacenter countries (by name), and status, privacy, and terms links. @@ -60,17 +61,18 @@ The out-of-the-box dashboards include an account **Overview**; **Model Compariso | **Workspace** | `GET /workspaces` | A workspace in the account. Management key only. | | **API Key** | `GET /keys` (per workspace) | An API key in a workspace, including its expiry date. Management key only. | -**Relationships:** each API Key records the ID of its Workspace. +Each API Key stores the ID of its Workspace as a property; no graph relationship is created between them. With a standard API key, only models and providers are imported; the workspace and API key import steps are skipped with a warning. ## Known limitations - **Benchmarks, rankings, app rankings, task classifications, and session costs refresh at most once a day** — OpenRouter limits these endpoints to 500 requests a day per account, so the plugin caches them for 24 hours. -- **Rankings licence** — rankings, app ranking, task classification, session cost, and benchmark data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Each row of these streams has a hidden **Citation** column with the exact attribution OpenRouter requires, and a hidden **As Of** date. The **Model Comparison**, **OpenRouter Ecosystem**, and **Model** dashboards credit the data already; show the citation on any dashboard you build from it. +- **Rankings licence** — rankings, app ranking, task classification, session cost, and benchmark data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Each row of these streams has a hidden **Citation** column with the exact attribution OpenRouter requires, and a hidden **As Of** date. The **Model Comparison**, **Ecosystem**, and **Model** dashboards credit the data already; show the citation on any dashboard you build from it. - **Not every model has benchmark scores** — Artificial Analysis scores cover fewer than half of models; the rest show blank. - **Usage history** can be queried up to a year back. Latency, time to first token, throughput, and grouping by **Provider**, **Origin**, **Country**, **Finish reason**, or **Streamed** are limited to 31 days; longer timeframes return an error. - **Usage timeframes start at 12 hours** — data is bucketed by hour, day, week, or month to suit the range; periods with no activity are left out rather than shown as zero. +- **Cost per 1M tokens is blended** — it's spend divided by all tokens, including cached and reasoning tokens, so it reflects your mix of traffic rather than a provider's list price. - **Large usage queries may be truncated** — the stream warns when this happens; narrow the timeframe or filter to specific objects. - **Model rankings start at 7 days** — the data is daily. Use case and language segments are weekly estimates from sampled traffic, so they're shown weekly even when **Day** is selected. - **Task classifications cover the last 7 days only** — they come from a sample of OpenRouter traffic, so only shares are available, not request or token counts. diff --git a/plugins/OpenRouter/v1/ui.json b/plugins/OpenRouter/v1/ui.json index e5b57f96..23698165 100644 --- a/plugins/OpenRouter/v1/ui.json +++ b/plugins/OpenRouter/v1/ui.json @@ -7,6 +7,6 @@ "validation": { "required": true }, - "help": "Use a [management key](https://openrouter.ai/settings/management-keys) for full access to workspace, API key, and usage data. A standard [API key](https://openrouter.ai/settings/keys) covers model, benchmark, ranking, and credit data only. This plugin only reads data; it does not create, modify, or delete anything in OpenRouter" + "help": "Use a [management key](https://openrouter.ai/settings/management-keys) for full access to workspace, API key, and usage data. A standard [API key](https://openrouter.ai/settings/keys) covers model, benchmark, ranking, and credit data only. This plugin only reads data; it does not create, modify, or delete anything in OpenRouter." } ] From edefa9353292e7d5986780a1d202de1d560d48a8 Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Tue, 6 Oct 2026 16:14:08 +0100 Subject: [PATCH 05/16] Score OpenRouter model prices as affordability Squashed from: - Score OpenRouter prices as affordability on the model radars - Lower the OpenRouter affordability ceiling default and minimum Co-Authored-By: Claude Opus 5.5 --- plugins/OpenRouter/v1/dataStreams/models.json | 40 +++++++++++++++++++ .../v1/dataStreams/scripts/models.js | 30 +++++++++++++- .../v1/defaultContent/model.dash.json | 20 ++++++---- .../defaultContent/modelComparison.dash.json | 20 ++++++---- plugins/OpenRouter/v1/docs/README.md | 2 +- 5 files changed, 96 insertions(+), 16 deletions(-) diff --git a/plugins/OpenRouter/v1/dataStreams/models.json b/plugins/OpenRouter/v1/dataStreams/models.json index 8cdec806..82b5fecb 100644 --- a/plugins/OpenRouter/v1/dataStreams/models.json +++ b/plugins/OpenRouter/v1/dataStreams/models.json @@ -91,6 +91,36 @@ "computed": true, "valueExpression": "{{ (v => v == null || v === '' || Number(v) < 0 ? null : Number(v) * 1e6)($['pricing.input_cache_read']) }}" }, + { + "name": "inputAffordability", + "displayName": "Input Affordability", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "outputAffordability", + "displayName": "Output Affordability", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "cacheReadAffordability", + "displayName": "Cache Read Affordability", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, { "name": "top_provider.max_completion_tokens", "displayName": "Max Completion Tokens", @@ -214,6 +244,16 @@ ] } } + }, + { + "type": "number", + "name": "affordabilityCeiling", + "label": "Affordability ceiling ($/1M tokens)", + "help": "Sets the price that scores 0 on the affordability columns. Prices score on a log scale, from 100 at $0.01 per 1M tokens down to 0 at this price.", + "defaultValue": 1000, + "validation": { + "min": 1 + } } ] } diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/models.js b/plugins/OpenRouter/v1/dataStreams/scripts/models.js index 265a97d9..5c134256 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/models.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/models.js @@ -5,5 +5,33 @@ const unwrap = (v) => (Array.isArray(v) ? v[0] : v); const selected = context.config?.models ?? []; const ids = new Set(selected.map((o) => unwrap(o.rawId)).filter(Boolean)); +// Affordability scores price per 1M tokens on a log scale so "higher is better" +// alongside the benchmark indices: 100 at FLOOR or less, 0 at the ceiling or more. +// The ceiling is the optional `affordabilityCeiling` parameter; an unset value or +// one under the field's minimum uses the default. Keep MIN_CEILING and +// DEFAULT_CEILING in sync with that field's validation.min and defaultValue. +const FLOOR = 0.01; +const MIN_CEILING = 1; +const DEFAULT_CEILING = 1000; +const requested = Number(context.config?.affordabilityCeiling); +const ceiling = Number.isFinite(requested) && requested >= MIN_CEILING ? requested : DEFAULT_CEILING; +const span = Math.log10(ceiling / FLOOR); + +// Per-token price string to a 0-100 score; null when missing, non-numeric or +// negative (variable pricing). Free models score Infinity, clamped to 100. +const affordability = (perToken) => { + const price = perToken == null || String(perToken).trim() === '' ? NaN : Number(perToken) * 1e6; + if (!Number.isFinite(price) || price < 0) { + return null; + } + + return Math.min(100, Math.max(0, (100 * Math.log10(ceiling / price)) / span)); +}; + const models = data?.data ?? []; -result = ids.size ? models.filter((m) => ids.has(m.id)) : models; +result = (ids.size ? models.filter((m) => ids.has(m.id)) : models).map((m) => ({ + ...m, + inputAffordability: affordability(m.pricing?.prompt), + outputAffordability: affordability(m.pricing?.completion), + cacheReadAffordability: affordability(m.pricing?.input_cache_read) +})); diff --git a/plugins/OpenRouter/v1/defaultContent/model.dash.json b/plugins/OpenRouter/v1/defaultContent/model.dash.json index 64a0ccc0..ec971f8c 100644 --- a/plugins/OpenRouter/v1/defaultContent/model.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/model.dash.json @@ -122,6 +122,9 @@ "pricing.prompt", "pricing.completion", "pricing.input_cache_read", + "inputAffordability", + "outputAffordability", + "cacheReadAffordability", "architecture.input_modalities", "architecture.output_modalities" ] @@ -1041,7 +1044,7 @@ "config": { "_type": "tile/data-stream", "title": "Benchmarks Vs Price", - "description": "", + "description": "Higher is better on every axis. Affordability scores price per 1M tokens on a log scale, from 100 at $0.01 or less to 0 at $1,000 or more. Change the ceiling with the Affordability ceiling parameter.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" @@ -1083,19 +1086,19 @@ "title": "Agentic Index" }, { - "field": "inputPrice", + "field": "inputAffordability", "type": "quantitative", - "title": "Input Price ($/1M Tokens)" + "title": "Input Affordability" }, { - "field": "outputPrice", + "field": "outputAffordability", "type": "quantitative", - "title": "Output Price ($/1M Tokens)" + "title": "Output Affordability" }, { - "field": "cacheReadPrice", + "field": "cacheReadAffordability", "type": "quantitative", - "title": "Cache Read Price ($/1M Tokens)" + "title": "Cache Read Affordability" } ] }, @@ -1133,6 +1136,9 @@ "pricing.prompt", "pricing.completion", "pricing.input_cache_read", + "inputAffordability", + "outputAffordability", + "cacheReadAffordability", "architecture.input_modalities", "architecture.output_modalities" ] diff --git a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json index f7f1dd57..1bf443c8 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json @@ -62,6 +62,9 @@ "pricing.prompt", "pricing.completion", "pricing.input_cache_read", + "inputAffordability", + "outputAffordability", + "cacheReadAffordability", "architecture.input_modalities", "architecture.output_modalities" ] @@ -700,7 +703,7 @@ "config": { "_type": "tile/data-stream", "title": "Top Agentic Models Compared", - "description": "", + "description": "Higher is better on every axis. Affordability scores price per 1M tokens on a log scale, from 100 at $0.01 or less to 0 at $1,000 or more. Change the ceiling with the Affordability ceiling parameter.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" @@ -784,19 +787,19 @@ "title": "Agentic Index" }, { - "field": "inputPrice", + "field": "inputAffordability", "type": "quantitative", - "title": "Input Price ($/1M Tokens)" + "title": "Input Affordability" }, { - "field": "outputPrice", + "field": "outputAffordability", "type": "quantitative", - "title": "Output Price ($/1M Tokens)" + "title": "Output Affordability" }, { - "field": "cacheReadPrice", + "field": "cacheReadAffordability", "type": "quantitative", - "title": "Cache Read Price ($/1M Tokens)" + "title": "Cache Read Affordability" } ] }, @@ -835,6 +838,9 @@ "pricing.prompt", "pricing.completion", "pricing.input_cache_read", + "inputAffordability", + "outputAffordability", + "cacheReadAffordability", "architecture.input_modalities", "architecture.output_modalities" ] diff --git a/plugins/OpenRouter/v1/docs/README.md b/plugins/OpenRouter/v1/docs/README.md index adabfd8b..e36d98fd 100644 --- a/plugins/OpenRouter/v1/docs/README.md +++ b/plugins/OpenRouter/v1/docs/README.md @@ -43,7 +43,7 @@ The out-of-the-box dashboards include an account **Overview**; **Model Compariso - **API Keys** — API keys in a workspace with spend and limits. Per workspace. Management key only. - **Workspaces** — workspaces in the account with their default models, provider sort, disabled server tools, whether BYOK spend counts toward budgets, and logging and broadcast settings, optionally filtered to selected workspaces. Account-wide. Management key only. - **Credits** — credits purchased, used, and remaining. Account-wide. -- **Models** — every model on OpenRouter with pricing ($ per 1M tokens), context length, modalities, and Artificial Analysis index scores, optionally filtered to selected models. +- **Models** — every model on OpenRouter with pricing ($ per 1M tokens), context length, modalities, Artificial Analysis index scores, and affordability scores (0 to 100, higher is cheaper), optionally filtered to selected models. Prices score on a log scale from 100 at $0.01 per 1M tokens to 0 at the **Affordability ceiling** parameter ($1,000 by default, minimum $1). - **Providers** — every provider on OpenRouter, with headquarters and datacenter countries (by name), and status, privacy, and terms links. - **Model Providers** — providers serving a model, with status, price (including cache reads), uptime over the last 5 minutes, 30 minutes, and day, and latency (p50 to p99) and throughput over the last 30 minutes. Per model. - **Benchmarks** — model benchmark scores from Artificial Analysis, Design Arena, and OpenRouter evals, one row per model and benchmark. From 258f6d59a37cab90069084df7b3627fd881585a9 Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 11:40:35 +0100 Subject: [PATCH 06/16] Add an OpenRouter Model Comparison dashboard beside Model Leaderboards Squashed from: - Rename the OpenRouter Model Comparison dashboard to Model Leaderboards - Add an OpenRouter Model Comparison dashboard for a chosen set of models - Add Design Arena, session cost and evals tiles to OpenRouter Model Comparison - Fix OpenRouter Elo chart range on Model Comparison - Use smartviz tables for the OpenRouter Model Comparison tables - Rename OpenRouter Side By Side tile to Models and format prices as text - Add provider and usage tiles to the OpenRouter Model Comparison dashboard - Document the OpenRouter Model Comparison dashboard in the README - De-duplicate selected models by slug in OpenRouter Model Comparison tiles - Use smartviz charts for the remaining OpenRouter Model Comparison tiles Co-Authored-By: Claude Opus 5.5 --- .../v1/defaultContent/manifest.json | 4 + .../defaultContent/modelComparison.dash.json | 1693 +++++++++++------ .../modelLeaderboards.dash.json | 874 +++++++++ .../OpenRouter/v1/defaultContent/scopes.json | 17 + plugins/OpenRouter/v1/docs/README.md | 6 +- 5 files changed, 2059 insertions(+), 535 deletions(-) create mode 100644 plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json diff --git a/plugins/OpenRouter/v1/defaultContent/manifest.json b/plugins/OpenRouter/v1/defaultContent/manifest.json index cc0d6c63..d572c741 100644 --- a/plugins/OpenRouter/v1/defaultContent/manifest.json +++ b/plugins/OpenRouter/v1/defaultContent/manifest.json @@ -4,6 +4,10 @@ "name": "overview", "type": "dashboard" }, + { + "name": "modelLeaderboards", + "type": "dashboard" + }, { "name": "modelComparison", "type": "dashboard" diff --git a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json index 1bf443c8..252eb8c0 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json @@ -2,17 +2,23 @@ "name": "Model Comparison", "schemaVersion": "1.5", "timeframe": "last30days", + "variables": [ + "{{variables.[Compared Models]}}" + ], "dashboard": { "_type": "layout/grid", - "columns": 6, + "columns": 4, "version": 1, "contents": [ { - "i": "0c29cf9b-46f4-478d-98c4-25a07523750b", + "i": "23cffe4d-9415-4150-b3ca-18786feac31b", "x": 0, "y": 0, - "w": 6, - "h": 4, + "w": 4, + "h": 3, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", "title": "Models", @@ -22,18 +28,171 @@ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[models]}}", - "name": "models", - "pluginConfigId": "{{configId}}", - "sort": { - "by": [ - [ - "benchmarks.artificial_analysis.intelligence_index", - "desc" - ] + "id": "datastream-sql", + "dataSourceConfig": { + "version": "2.0", + "sql": "SELECT\n s.sourceId AS id,\n m.name,\n m.author,\n m.inputPrice,\n m.outputPrice,\n m.cacheReadPrice,\n m.context_length AS contextLength,\n m.\"top_provider.max_completion_tokens\" AS maxCompletionTokens,\n m.inputModalities,\n m.outputModalities,\n m.\"benchmarks.artificial_analysis.intelligence_index\" AS intelligenceIndex,\n m.\"benchmarks.artificial_analysis.coding_index\" AS codingIndex,\n m.\"benchmarks.artificial_analysis.agentic_index\" AS agenticIndex,\n m.knowledge_cutoff AS knowledgeCutoff,\n m.created,\n m.expiration_date AS expirationDate\nFROM Models m\nJOIN Selected s ON m.id = s.rawId\nORDER BY m.name", + "tables": [ + { + "tableName": "Models", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } + }, + { + "tableName": "Selected", + "config": { + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], + "timeframe": "none" + } + } ] - } + }, + "metadata": [ + { + "name": "id", + "displayName": "Model ID", + "shape": "string", + "visible": false + }, + { + "name": "name", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "sourceId": "id", + "sourceType": "Model", + "name": "name" + }, + { + "name": "author", + "displayName": "Author", + "shape": "string" + }, + { + "name": "inputPrice", + "displayName": "Input Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "name": "outputPrice", + "displayName": "Output Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "name": "cacheReadPrice", + "displayName": "Cache Read Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "name": "contextLength", + "displayName": "Context Length", + "shape": "number" + }, + { + "name": "maxCompletionTokens", + "displayName": "Max Completion Tokens", + "shape": "number" + }, + { + "name": "inputModalities", + "displayName": "Input Modalities", + "shape": "string" + }, + { + "name": "outputModalities", + "displayName": "Output Modalities", + "shape": "string" + }, + { + "name": "intelligenceIndex", + "displayName": "Intelligence Index", + "shape": "number" + }, + { + "name": "codingIndex", + "displayName": "Coding Index", + "shape": "number" + }, + { + "name": "agenticIndex", + "displayName": "Agentic Index", + "shape": "number" + }, + { + "name": "knowledgeCutoff", + "displayName": "Knowledge Cutoff", + "shape": [ + "date", + { + "timeZone": "Etc/UTC" + } + ] + }, + { + "name": "created", + "displayName": "Created", + "shape": [ + "date", + { + "inputPattern": "t" + } + ] + }, + { + "name": "expirationDate", + "displayName": "Expiration Date", + "shape": [ + "date", + { + "timeZone": "Etc/UTC" + } + ] + } + ] }, + "variables": [ + "{{variables.[Compared Models]}}" + ], "visualisation": { "type": "data-stream-table", "config": { @@ -44,824 +203,1294 @@ "author", "inputPrice", "outputPrice", - "context_length", - "benchmarks.artificial_analysis.intelligence_index", - "benchmarks.artificial_analysis.coding_index", - "benchmarks.artificial_analysis.agentic_index" - ], - "hiddenColumns": [ - "id", - "canonical_slug", "cacheReadPrice", - "top_provider.max_completion_tokens", + "contextLength", + "maxCompletionTokens", "inputModalities", "outputModalities", - "knowledge_cutoff", + "intelligenceIndex", + "codingIndex", + "agenticIndex", + "knowledgeCutoff", "created", - "expiration_date", - "pricing.prompt", - "pricing.completion", - "pricing.input_cache_read", - "inputAffordability", - "outputAffordability", - "cacheReadAffordability", - "architecture.input_modalities", - "architecture.output_modalities" - ] + "expirationDate" + ], + "hiddenColumns": [ + "id" + ], + "rowLinkColumnName": "name" } } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "name", + "title": "Model", + "renderer": { + "type": "text" + } + }, + { + "field": "author", + "title": "Author", + "renderer": { + "type": "text" + } + }, + { + "field": "inputPrice", + "title": "Input Price ($/1M Tokens)", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "outputPrice", + "title": "Output Price ($/1M Tokens)", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "cacheReadPrice", + "title": "Cache Read Price ($/1M Tokens)", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "contextLength", + "title": "Context Length", + "align": "right", + "renderer": { + "type": "number", + "compact": true + } + }, + { + "field": "maxCompletionTokens", + "title": "Max Completion Tokens", + "align": "right", + "renderer": { + "type": "number", + "compact": true + } + }, + { + "field": "inputModalities", + "title": "Input Modalities", + "renderer": { + "type": "text" + } + }, + { + "field": "outputModalities", + "title": "Output Modalities", + "renderer": { + "type": "text" + } + }, + { + "field": "intelligenceIndex", + "title": "Intelligence Index", + "align": "right", + "renderer": { + "type": "number", + "decimals": 2 + } + }, + { + "field": "codingIndex", + "title": "Coding Index", + "align": "right", + "renderer": { + "type": "number", + "decimals": 2 + } + }, + { + "field": "agenticIndex", + "title": "Agentic Index", + "align": "right", + "renderer": { + "type": "number", + "decimals": 2 + } + } + ] + }, + "style": {} + } } } }, { - "i": "3eec66c4-de77-40ee-b403-37620ac6ec6b", + "i": "ea3b78fb-bb19-4538-a408-5d066f96ab52", "x": 0, - "y": 4, + "y": 3, "w": 2, - "h": 3, + "h": 5, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", - "title": "Cost Per Task For Top GPQA Models", - "description": "By Accuracy", + "title": "Benchmarks Vs Price", + "description": "Higher is better on every axis. Affordability scores price per 1M tokens on a log scale, from 100 at $0.01 or less to 0 at $1,000 or more.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[benchmarks]}}", - "name": "benchmarks", - "pluginConfigId": "{{configId}}", + "id": "datastream-sql", "dataSourceConfig": { - "source": "openrouter" - }, - "filter": { - "multiOperation": "and", - "filters": [ + "version": "2.0", + "sql": "SELECT\n s.sourceId AS id,\n m.name,\n m.\"benchmarks.artificial_analysis.intelligence_index\" AS intelligenceIndex,\n m.\"benchmarks.artificial_analysis.coding_index\" AS codingIndex,\n m.\"benchmarks.artificial_analysis.agentic_index\" AS agenticIndex,\n m.inputAffordability,\n m.outputAffordability,\n m.cacheReadAffordability\nFROM Models m\nJOIN Selected s ON m.id = s.rawId\nORDER BY m.name", + "tables": [ { - "column": "benchmark", - "operation": "equals", - "value": "gpqa_diamond" + "tableName": "Models", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } }, { - "column": "metric", - "operation": "equals", - "value": "accuracy" + "tableName": "Selected", + "config": { + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], + "timeframe": "none" + } } ] }, - "sort": { - "by": [ - [ - "score", - "desc" + "metadata": [ + { + "name": "id", + "displayName": "Model ID", + "shape": "string", + "visible": false + }, + { + "name": "name", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "intelligenceIndex", + "displayName": "Intelligence Index", + "shape": "number" + }, + { + "name": "codingIndex", + "displayName": "Coding Index", + "shape": "number" + }, + { + "name": "agenticIndex", + "displayName": "Agentic Index", + "shape": "number" + }, + { + "name": "inputAffordability", + "displayName": "Input Affordability", + "shape": [ + "number", + { + "decimalPlaces": 0 + } ] - ], - "top": 15 + }, + { + "name": "outputAffordability", + "displayName": "Output Affordability", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "cacheReadAffordability", + "displayName": "Cache Read Affordability", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + } + ] + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "radar", + "encoding": { + "series": { + "field": "name", + "type": "nominal", + "title": "Model" + }, + "axes": [ + { + "field": "intelligenceIndex", + "type": "quantitative", + "title": "Intelligence Index" + }, + { + "field": "codingIndex", + "type": "quantitative", + "title": "Coding Index" + }, + { + "field": "agenticIndex", + "type": "quantitative", + "title": "Agentic Index" + }, + { + "field": "inputAffordability", + "type": "quantitative", + "title": "Input Affordability" + }, + { + "field": "outputAffordability", + "type": "quantitative", + "title": "Output Affordability" + }, + { + "field": "cacheReadAffordability", + "type": "quantitative", + "title": "Cache Read Affordability" + } + ] + }, + "style": { + "legendPosition": "right", + "radarShape": "polygon", + "radarFill": false, + "radarRadius": 80 + } } }, "visualisation": { - "type": "data-stream-bar-chart", + "type": "data-stream-table", "config": { - "data-stream-bar-chart": { - "xAxisData": "modelName", - "yAxisData": [ - "avgCostPerTask" - ], - "xAxisGroup": "none", - "xAxisLabel": "Model", - "yAxisLabel": "Avg Cost Per Task (USD)", - "showXAxisLabel": true, - "showYAxisLabel": true, - "showLegend": false, - "legendPosition": "bottom", - "showGrid": true, - "horizontalLayout": "horizontal", - "displayMode": "actual", - "showTotals": false, - "showValue": false, - "grouping": false, - "range": { - "type": "auto" - } + "data-stream-table": { + "hiddenColumns": [ + "id" + ] } } } } }, { - "i": "df0c3dda-bfa5-48a0-babf-ed96590b4be2", + "i": "e8e0ca65-ced6-4265-b2ac-74fea2e224f0", "x": 2, - "y": 4, + "y": 3, "w": 2, - "h": 3, + "h": 5, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", - "title": "Top Models For Website Design (Elo)", - "description": "By Elo Rating (Design Arena)", + "title": "Artificial Analysis Indices", + "description": "", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[benchmarks]}}", - "name": "benchmarks", - "pluginConfigId": "{{configId}}", + "id": "datastream-sql", "dataSourceConfig": { - "source": "design-arena" - }, - "filter": { - "multiOperation": "and", - "filters": [ + "version": "2.0", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug,\n sourceId,\n name\n FROM Selected\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n s.name,\n b.benchmark,\n b.score\nFROM Benchmarks b\nJOIN by_slug s ON b.model = s.canonical_slug\nWHERE b.source = 'artificial-analysis'\nORDER BY b.benchmark, s.name", + "tables": [ { - "column": "benchmark", - "operation": "equals", - "value": "website" + "tableName": "Benchmarks", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } }, { - "column": "metric", - "operation": "equals", - "value": "elo" + "tableName": "Selected", + "config": { + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], + "timeframe": "none" + } } ] }, - "sort": { - "by": [ - [ - "score", - "desc" + "metadata": [ + { + "name": "name", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "benchmark", + "displayName": "Benchmark", + "shape": "string" + }, + { + "name": "score", + "displayName": "Score", + "shape": [ + "number", + { + "decimalPlaces": 1 + } ] - ], - "top": 15 - } + } + ] }, + "variables": [ + "{{variables.[Compared Models]}}" + ], "visualisation": { "type": "data-stream-bar-chart", "config": { "data-stream-bar-chart": { - "xAxisData": "modelName", + "xAxisData": "benchmark", "yAxisData": [ "score" ], - "xAxisGroup": "none", - "xAxisLabel": "Model", - "yAxisLabel": "Elo", - "showXAxisLabel": true, + "xAxisGroup": "name", + "xAxisLabel": "", + "yAxisLabel": "Index", + "showXAxisLabel": false, "showYAxisLabel": true, - "showLegend": false, + "showLegend": true, "legendPosition": "bottom", "showGrid": true, - "horizontalLayout": "horizontal", + "horizontalLayout": "vertical", "displayMode": "actual", "showTotals": false, "showValue": false, - "grouping": false, + "grouping": true, "range": { "type": "auto" } } } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "benchmark", + "type": "nominal", + "title": "Benchmark" + }, + "y": { + "field": "score", + "type": "quantitative", + "title": "Score" + }, + "series": { + "field": "name", + "type": "nominal", + "title": "Model" + } + }, + "style": { + "cartesianStack": "off", + "legendPosition": "bottom", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Index" + } + } } } }, { - "i": "3609a3d6-a8ff-470d-8333-20744922db53", - "x": 4, - "y": 4, + "i": "e4a900b8-db8c-4f61-ab49-a272d80fdd2f", + "x": 0, + "y": 8, "w": 2, - "h": 3, + "h": 5, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", - "title": "Top Models By Intelligence Index", - "description": "By Index Score (Artificial Analysis)", + "title": "Design Arena Elo", + "description": "Elo rating by design category. Higher is better.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[benchmarks]}}", - "name": "benchmarks", - "pluginConfigId": "{{configId}}", + "id": "datastream-sql", "dataSourceConfig": { - "source": "artificial-analysis" - }, - "filter": { - "multiOperation": "and", - "filters": [ + "version": "2.0", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug,\n sourceId,\n name\n FROM Selected\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n s.name,\n b.benchmark,\n b.score\nFROM Benchmarks b\nJOIN by_slug s ON b.model = s.canonical_slug\nWHERE b.source = 'design-arena' AND b.metric = 'elo'\nORDER BY b.benchmark, s.name", + "tables": [ { - "column": "benchmark", - "operation": "equals", - "value": "Intelligence Index" + "tableName": "Benchmarks", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } }, { - "column": "metric", - "operation": "equals", - "value": "index" + "tableName": "Selected", + "config": { + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], + "timeframe": "none" + } } ] }, - "sort": { - "by": [ - [ - "score", - "desc" + "metadata": [ + { + "name": "name", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "benchmark", + "displayName": "Category", + "shape": "string" + }, + { + "name": "score", + "displayName": "Elo", + "shape": [ + "number", + { + "decimalPlaces": 0 + } ] - ], - "top": 15 - } + } + ] }, + "variables": [ + "{{variables.[Compared Models]}}" + ], "visualisation": { "type": "data-stream-bar-chart", "config": { "data-stream-bar-chart": { - "xAxisData": "modelName", + "xAxisData": "benchmark", "yAxisData": [ "score" ], - "xAxisGroup": "none", - "xAxisLabel": "Model", - "yAxisLabel": "Intelligence Index", - "showXAxisLabel": true, + "xAxisGroup": "name", + "xAxisLabel": "", + "yAxisLabel": "Elo", + "showXAxisLabel": false, "showYAxisLabel": true, - "showLegend": false, + "showLegend": true, "legendPosition": "bottom", "showGrid": true, - "horizontalLayout": "horizontal", + "horizontalLayout": "vertical", "displayMode": "actual", "showTotals": false, "showValue": false, - "grouping": false, + "grouping": true, "range": { - "type": "auto" + "type": "custom", + "maximum": 1600, + "minimum": 900 + } + } + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "benchmark", + "type": "nominal", + "title": "Category" + }, + "y": { + "field": "score", + "type": "quantitative", + "title": "Elo" + }, + "series": { + "field": "name", + "type": "nominal", + "title": "Model" } + }, + "style": { + "cartesianStack": "off", + "legendPosition": "bottom", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Elo", + "cartesianYAxisMin": 900, + "cartesianYAxisMax": 1600 } } } } }, { - "i": "bdbfd6a7-4bbe-4eab-91b5-6908a10344d7", - "x": 0, - "y": 7, - "w": 3, - "h": 3, + "i": "4bf8b04e-889d-4518-b96c-491fe668c142", + "x": 2, + "y": 8, + "w": 2, + "h": 5, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", - "title": "Most Used Models On OpenRouter", - "description": "", + "title": "Session Cost By Harness", + "description": "Median cost of one 10–49 turn coding-agent session, by model and harness.", + "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[rankings]}}", - "name": "rankings", - "pluginConfigId": "{{configId}}", - "filter": { - "multiOperation": "and", - "filters": [ + "id": "datastream-sql", + "dataSourceConfig": { + "version": "2.0", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug,\n sourceId,\n name\n FROM Selected\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n s.name,\n c.app_name AS harness,\n c.median_session_cost_usd AS cost\nFROM SessionCost c\nJOIN by_slug s ON c.model_permaslug = s.canonical_slug\nWHERE c.turn_range = '10-49-turns'\nORDER BY s.name, c.app_name", + "tables": [ { - "column": "model_permaslug", - "operation": "notequals", - "value": "other" - } - ] - }, - "group": { - "by": [ - [ - "model_permaslug", - "uniqueValues" - ] - ], - "aggregate": [ + "tableName": "SessionCost", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[sessionCost]}}", + "name": "sessionCost", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } + }, { - "type": "sum", - "names": [ - "total_tokens" - ] + "tableName": "Selected", + "config": { + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], + "timeframe": "none" + } } ] }, - "sort": { - "by": [ - [ - "total_tokens_sum", - "desc" - ] - ], - "top": 10 - }, "metadata": [ { - "name": "total_tokens_sum", + "name": "name", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "harness", + "displayName": "Harness", + "shape": "string" + }, + { + "name": "cost", + "displayName": "Median Session Cost", "shape": [ - "number", + "currency", { - "decimalPlaces": 0, - "thousandsSeparator": true + "code": "USD", + "decimalPlaces": 4 } ] - }, - { - "pattern": ".*" } ] }, - "visualisation": { - "type": "data-stream-bar-chart", - "config": { - "data-stream-bar-chart": { - "xAxisData": "model_permaslug_uniqueValues", - "yAxisData": [ - "total_tokens_sum" - ], - "xAxisGroup": "none", - "xAxisLabel": "Model", - "yAxisLabel": "Tokens", - "showXAxisLabel": true, - "showYAxisLabel": true, - "showLegend": false, - "legendPosition": "bottom", - "showGrid": true, - "horizontalLayout": "horizontal", - "displayMode": "actual", - "showTotals": false, - "showValue": false, - "grouping": false, - "range": { - "type": "auto" + "variables": [ + "{{variables.[Compared Models]}}" + ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "scatter", + "encoding": { + "x": { + "field": "name", + "type": "nominal", + "title": "Model" + }, + "y": { + "field": "cost", + "type": "quantitative", + "title": "Median Session Cost ($)" + }, + "color": { + "field": "harness", + "type": "nominal", + "title": "Harness" } + }, + "style": { + "legendPosition": "bottom", + "cartesianYAxisScale": "log", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Median Session Cost ($, log scale)", + "cartesianShowVerticalGridLines": true, + "cartesianXAxisLabelRotation": 45, + "cartesianXAxisTickDensity": "dense" } } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": {} + } } } }, { - "i": "22bac160-9e66-4597-b230-270811258056", - "x": 3, - "y": 7, - "w": 3, - "h": 3, + "i": "2c1f928b-b7a2-48d2-ac16-e100b0f74f21", + "x": 0, + "y": 13, + "w": 4, + "h": 4, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", - "title": "OpenRouter Token Share By Author", - "description": "", + "title": "OpenRouter Evals", + "description": "Results from OpenRouter's own evals. The metric differs by benchmark, so scores aren't comparable across rows.", + "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[rankings]}}", - "name": "rankings", - "pluginConfigId": "{{configId}}", - "filter": { - "multiOperation": "and", - "filters": [ + "id": "datastream-sql", + "dataSourceConfig": { + "version": "2.0", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug,\n sourceId,\n name\n FROM Selected\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n s.sourceId AS id,\n s.name,\n b.benchmark,\n b.metric,\n b.score,\n b.tasks,\n b.avgCostPerTask,\n b.avgDurationMs\nFROM Benchmarks b\nJOIN by_slug s ON b.model = s.canonical_slug\nWHERE b.source = 'openrouter'\nORDER BY b.score DESC", + "tables": [ { - "column": "model_permaslug", - "operation": "notequals", - "value": "other" - } - ] - }, - "group": { - "by": [ - [ - "author", - "uniqueValues" - ] - ], - "aggregate": [ + "tableName": "Benchmarks", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } + }, { - "type": "sum", - "names": [ - "total_tokens" - ] + "tableName": "Selected", + "config": { + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], + "timeframe": "none" + } } ] }, - "sort": { - "by": [ - [ - "total_tokens_sum", - "desc" - ] - ] - }, "metadata": [ { - "name": "total_tokens_sum", + "name": "id", + "displayName": "Model ID", + "shape": "string", + "visible": false + }, + { + "name": "name", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "sourceId": "id", + "sourceType": "Model", + "name": "name" + }, + { + "name": "benchmark", + "displayName": "Benchmark", + "shape": "string" + }, + { + "name": "metric", + "displayName": "Metric", + "shape": "string" + }, + { + "name": "score", + "displayName": "Score", "shape": [ "number", { - "decimalPlaces": 0, - "thousandsSeparator": true + "decimalPlaces": 2 } ] }, { - "pattern": ".*" + "name": "tasks", + "displayName": "Tasks", + "shape": "number" + }, + { + "name": "avgCostPerTask", + "displayName": "Avg Cost per Task", + "shape": "usd" + }, + { + "name": "avgDurationMs", + "displayName": "Avg Duration", + "shape": "milliseconds" } ] }, + "variables": [ + "{{variables.[Compared Models]}}" + ], "visualisation": { - "type": "data-stream-donut-chart", + "type": "data-stream-table", "config": { - "data-stream-donut-chart": { - "valueColumn": "total_tokens_sum", - "labelColumn": "author_uniqueValues", - "hideCenterValue": false, - "showValuesAsPercentage": true, - "legendPosition": "auto", - "legendMode": "table" + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "name", + "benchmark", + "metric", + "score", + "tasks", + "avgCostPerTask", + "avgDurationMs" + ], + "hiddenColumns": [ + "id" + ], + "rowLinkColumnName": "name" } } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "name", + "title": "Model", + "renderer": { + "type": "text" + } + }, + { + "field": "benchmark", + "title": "Benchmark", + "renderer": { + "type": "text" + } + }, + { + "field": "metric", + "title": "Metric", + "renderer": { + "type": "text" + } + }, + { + "field": "score", + "title": "Score", + "align": "right", + "renderer": { + "type": "number", + "decimals": 2 + } + }, + { + "field": "tasks", + "title": "Tasks", + "align": "right", + "renderer": { + "type": "number", + "decimals": 0 + } + }, + { + "field": "avgCostPerTask", + "title": "Avg Cost per Task", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "avgDurationMs", + "title": "Avg Duration", + "align": "right", + "renderer": { + "type": "number", + "decimals": 0 + } + } + ] + }, + "style": {} + } } } }, { - "i": "071c926f-0d3c-4192-962f-d191aa03dcbc", + "i": "f835faa9-edd8-4bbb-8f89-659d4eadd362", "x": 0, - "y": 10, - "w": 6, - "h": 4, + "y": 17, + "w": 2, + "h": 5, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", - "title": "Top Models On OpenRouter By Week (This Year)", - "description": "Each week's 20 most used models across OpenRouter, by tokens. All other models in that week are combined into Other.", - "timeframe": "thisYear", + "title": "Throughput By Provider", + "description": "Median tokens per second over the last 30 minutes.", + "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[rankings]}}", - "name": "rankings", + "id": "{{dataStreams.[modelProviders]}}", + "name": "modelProviders", "pluginConfigId": "{{configId}}", - "dataSourceConfig": { - "period": "week", - "top": 20 - }, "sort": { "by": [ [ - "date", - "asc" + "throughput_last_30m.p50", + "desc" ] ] } }, - "vizSpec": { - "version": "1.0", - "visualization": { - "chartType": "bar", - "encoding": { - "x": { - "field": "date", - "type": "temporal", - "title": "Week" - }, - "y": { - "field": "total_tokens", - "type": "quantitative", - "aggregate": "sum", - "title": "Tokens" - }, - "series": { - "field": "model_permaslug", - "type": "nominal", - "title": "Model" - } - }, - "style": { - "cartesianStack": "on", - "legendPosition": "bottom" - } - } + "scope": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" }, + "variables": [ + "{{variables.[Compared Models]}}" + ], "visualisation": { - "type": "data-stream-line-graph", + "type": "data-stream-bar-chart", "config": { - "data-stream-line-graph": { - "xAxisColumn": "date", - "yAxisColumn": [ - "total_tokens" + "data-stream-bar-chart": { + "xAxisData": "tag", + "yAxisData": [ + "throughput_last_30m.p50" ], - "seriesColumn": "model_permaslug", + "xAxisGroup": "model_name", + "xAxisLabel": "Endpoint", + "yAxisLabel": "Tokens/s", + "showXAxisLabel": false, + "showYAxisLabel": true, "showLegend": true, "legendPosition": "bottom", - "yAxisLabel": "Tokens", - "showYAxisLabel": true, - "showTrendLine": false + "showGrid": true, + "horizontalLayout": "vertical", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": true, + "range": { + "type": "auto" + } } } } } }, { - "i": "a3384d79-77a8-49c0-bb08-62d6744303e1", - "x": 0, - "y": 14, - "w": 6, - "h": 4, + "i": "88a7859f-3bb2-42ba-9a83-9beaaae99d05", + "x": 2, + "y": 17, + "w": 2, + "h": 5, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", - "title": "Top Model By Task", - "description": "", + "title": "Latency By Provider", + "description": "Median latency over the last 30 minutes.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[taskClassifications]}}", - "name": "taskClassifications", + "id": "{{dataStreams.[modelProviders]}}", + "name": "modelProviders", "pluginConfigId": "{{configId}}", "sort": { "by": [ [ - "usageShare", - "desc" + "latency_last_30m.p50", + "asc" ] ] } }, + "scope": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], "visualisation": { - "type": "data-stream-table", + "type": "data-stream-bar-chart", "config": { - "data-stream-table": { - "transpose": false, - "columnOrder": [ - "task", - "category", - "usageShare", - "topModel", - "topModelUsageShare", - "asOf" + "data-stream-bar-chart": { + "xAxisData": "tag", + "yAxisData": [ + "latency_last_30m.p50" ], - "hiddenColumns": [ - "tag", - "tokenShare", - "citation" - ] + "xAxisGroup": "model_name", + "xAxisLabel": "Endpoint", + "yAxisLabel": "Latency (ms)", + "showXAxisLabel": false, + "showYAxisLabel": true, + "showLegend": true, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "vertical", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": true, + "range": { + "type": "auto" + } } } } } }, { - "i": "37f6bcdf-02a2-4c2a-861d-c9515d1c2092", + "i": "cd3d1575-854e-4b32-bdf0-501ddf31ae99", "x": 0, - "y": 18, - "w": 6, - "h": 8, + "y": 22, + "w": 2, + "h": 4, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", - "title": "Claude Code Cost Per Session", - "description": "Median cost of one Claude Code session routed through OpenRouter, by model and session length. Users can pick any model on OpenRouter, so many aren't Anthropic's.", - "timeframe": "none", + "title": "Your Spend (Last 30 Days)", + "description": "Your own spend on each model through your OpenRouter account.", + "timeframe": "last30days", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[sessionCost]}}", - "name": "sessionCost", + "id": "{{dataStreams.[usage]}}", + "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "harness": "claude-code" + "metric": "total_usage", + "groupBy": "model", + "filterObjects": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" + } }, "sort": { "by": [ [ - "sessionLengthOrder", - "asc" - ], - [ - "median_session_cost_usd", + "date", "asc" ] ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "group", + "showLegend": true, + "legendPosition": "bottom", + "yAxisLabel": "Spend ($)", + "showYAxisLabel": true, + "showTrendLine": false + } } }, "vizSpec": { "version": "1.0", "visualization": { - "chartType": "scatter", + "chartType": "line", "encoding": { "x": { - "field": "model_permaslug", - "type": "nominal", - "title": "Model" + "field": "date", + "type": "temporal", + "title": "Date" }, "y": { - "field": "median_session_cost_usd", + "field": "value", "type": "quantitative", - "title": "Median Session Cost ($)" + "title": "Spend ($)" }, - "color": { - "field": "sessionLength", + "series": { + "field": "group", "type": "nominal", - "title": "Session Length" + "title": "Model" } }, "style": { "legendPosition": "bottom", - "cartesianYAxisScale": "log", - "cartesianXAxisLabelRotation": 45, - "cartesianXAxisTickDensity": "dense", "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Median Session Cost ($, log scale)", - "cartesianShowVerticalGridLines": true - } - } - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { - "transpose": false, - "columnOrder": [ - "model_permaslug", - "sessionLength", - "median_session_cost_usd", - "app_name" - ], - "hiddenColumns": [ - "app_slug", - "turn_range", - "sessionLengthOrder", - "asOf", - "citation" - ] + "cartesianYAxisTitle": "Spend ($)" } } } } }, { - "i": "56d4d9f8-9cc4-4852-9306-2997d9e87000", - "x": 0, - "y": 26, - "w": 6, - "h": 8, + "i": "97618c1f-a773-45f4-bba5-ccd2afa7526a", + "x": 2, + "y": 22, + "w": 2, + "h": 4, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/data-stream", - "title": "Top Agentic Models Compared", - "description": "Higher is better on every axis. Affordability scores price per 1M tokens on a log scale, from 100 at $0.01 or less to 0 at $1,000 or more. Change the ceiling with the Affordability ceiling parameter.", - "timeframe": "none", + "title": "Your Cost Per 1M Tokens (Last 30 Days)", + "description": "Your own cost per million tokens on each model.", + "timeframe": "last30days", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[models]}}", - "name": "models", + "id": "{{dataStreams.[usage]}}", + "name": "usage", "pluginConfigId": "{{configId}}", - "filter": { - "multiOperation": "and", - "filters": [ - { - "column": "benchmarks.artificial_analysis.agentic_index", - "operation": "greaterthan", - "value": "0" - }, - { - "column": "benchmarks.artificial_analysis.intelligence_index", - "operation": "greaterthan", - "value": "0" - }, - { - "column": "benchmarks.artificial_analysis.coding_index", - "operation": "greaterthan", - "value": "0" - }, - { - "column": "name", - "operation": "notcontains", - "value": "(batch)" - }, - { - "column": "name", - "operation": "notcontains", - "value": "(free)" - } - ] + "dataSourceConfig": { + "metric": "cost_per_million_tokens", + "groupBy": "model", + "filterObjects": { + "scope": "{{scopes.[Model Comparison]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Compared Models]}}" + } }, "sort": { "by": [ [ - "benchmarks.artificial_analysis.agentic_index", - "desc" - ], - [ - "benchmarks.artificial_analysis.intelligence_index", - "desc" - ], - [ - "benchmarks.artificial_analysis.coding_index", - "desc" + "date", + "asc" + ] + ] + }, + "metadata": [ + { + "name": "value", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 4 + } ] - ], - "top": 10 + }, + { + "pattern": ".*" + } + ] + }, + "variables": [ + "{{variables.[Compared Models]}}" + ], + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "value" + ], + "seriesColumn": "group", + "showLegend": true, + "legendPosition": "bottom", + "yAxisLabel": "Cost per 1M tokens ($)", + "showYAxisLabel": true, + "showTrendLine": false + } } }, "vizSpec": { "version": "1.0", "visualization": { - "chartType": "radar", + "chartType": "line", "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Cost per 1M tokens ($)" + }, "series": { - "field": "name", + "field": "group", "type": "nominal", "title": "Model" - }, - "axes": [ - { - "field": "benchmarks.artificial_analysis.intelligence_index", - "type": "quantitative", - "title": "Intelligence Index" - }, - { - "field": "benchmarks.artificial_analysis.coding_index", - "type": "quantitative", - "title": "Coding Index" - }, - { - "field": "benchmarks.artificial_analysis.agentic_index", - "type": "quantitative", - "title": "Agentic Index" - }, - { - "field": "inputAffordability", - "type": "quantitative", - "title": "Input Affordability" - }, - { - "field": "outputAffordability", - "type": "quantitative", - "title": "Output Affordability" - }, - { - "field": "cacheReadAffordability", - "type": "quantitative", - "title": "Cache Read Affordability" - } - ] + } }, "style": { - "legendPosition": "right", - "radarShape": "polygon", - "radarFill": false, - "radarRadius": 80 - } - } - }, - "visualisation": { - "type": "data-stream-table", - "config": { - "data-stream-table": { - "columnOrder": [ - "name", - "author", - "benchmarks.artificial_analysis.intelligence_index", - "benchmarks.artificial_analysis.coding_index", - "benchmarks.artificial_analysis.agentic_index", - "inputPrice", - "outputPrice", - "cacheReadPrice" - ], - "hiddenColumns": [ - "id", - "canonical_slug", - "cacheReadPrice", - "top_provider.max_completion_tokens", - "inputModalities", - "outputModalities", - "knowledge_cutoff", - "created", - "expiration_date", - "pricing.prompt", - "pricing.completion", - "pricing.input_cache_read", - "inputAffordability", - "outputAffordability", - "cacheReadAffordability", - "architecture.input_modalities", - "architecture.output_modalities" - ] + "legendPosition": "bottom", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Cost per 1M tokens ($)" } } } } }, { - "i": "679f0166-77c4-47c1-9ae5-7e4c1721f718", + "i": "e26858ee-6975-4e02-b7ee-c8b2b46614b0", "x": 0, - "y": 34, - "w": 6, + "y": 26, + "w": 4, "h": 1, + "moved": false, + "static": false, + "z": 0, "config": { "_type": "tile/text", "title": "", "description": "", "visualisation": { "config": { - "content": "Benchmarks: Artificial Analysis ([artificialanalysis.ai](https://artificialanalysis.ai)), Design Arena ([www.designarena.ai](https://www.designarena.ai)), and OpenRouter evals ([openrouter.ai](https://openrouter.ai)), via OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). \nRankings, tasks, and session cost: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). Task data is as of the **As Of** date in the Top Model By Task table; session cost is refreshed weekly. \nLicensed under CC BY 4.0.", + "content": "Benchmarks: Artificial Analysis ([artificialanalysis.ai](https://artificialanalysis.ai)), Design Arena ([www.designarena.ai](https://www.designarena.ai)), and OpenRouter evals ([openrouter.ai](https://openrouter.ai)), via OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). \nSession cost: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)), refreshed weekly. \nLicensed under CC BY 4.0.", "fontSize": 12, "align": "left", "autoSize": true diff --git a/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json b/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json new file mode 100644 index 00000000..5f8c783b --- /dev/null +++ b/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json @@ -0,0 +1,874 @@ +{ + "name": "Model Leaderboards", + "schemaVersion": "1.5", + "timeframe": "last30days", + "dashboard": { + "_type": "layout/grid", + "columns": 6, + "version": 1, + "contents": [ + { + "i": "0c29cf9b-46f4-478d-98c4-25a07523750b", + "x": 0, + "y": 0, + "w": 6, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Models", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "benchmarks.artificial_analysis.intelligence_index", + "desc" + ] + ] + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "name", + "author", + "inputPrice", + "outputPrice", + "context_length", + "benchmarks.artificial_analysis.intelligence_index", + "benchmarks.artificial_analysis.coding_index", + "benchmarks.artificial_analysis.agentic_index" + ], + "hiddenColumns": [ + "id", + "canonical_slug", + "cacheReadPrice", + "top_provider.max_completion_tokens", + "inputModalities", + "outputModalities", + "knowledge_cutoff", + "created", + "expiration_date", + "pricing.prompt", + "pricing.completion", + "pricing.input_cache_read", + "inputAffordability", + "outputAffordability", + "cacheReadAffordability", + "architecture.input_modalities", + "architecture.output_modalities" + ] + } + } + } + } + }, + { + "i": "3eec66c4-de77-40ee-b403-37620ac6ec6b", + "x": 0, + "y": 4, + "w": 2, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "Cost Per Task For Top GPQA Models", + "description": "By Accuracy", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "source": "openrouter" + }, + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "benchmark", + "operation": "equals", + "value": "gpqa_diamond" + }, + { + "column": "metric", + "operation": "equals", + "value": "accuracy" + } + ] + }, + "sort": { + "by": [ + [ + "score", + "desc" + ] + ], + "top": 15 + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "modelName", + "yAxisData": [ + "avgCostPerTask" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Avg Cost Per Task (USD)", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "df0c3dda-bfa5-48a0-babf-ed96590b4be2", + "x": 2, + "y": 4, + "w": 2, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "Top Models For Website Design (Elo)", + "description": "By Elo Rating (Design Arena)", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "source": "design-arena" + }, + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "benchmark", + "operation": "equals", + "value": "website" + }, + { + "column": "metric", + "operation": "equals", + "value": "elo" + } + ] + }, + "sort": { + "by": [ + [ + "score", + "desc" + ] + ], + "top": 15 + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "modelName", + "yAxisData": [ + "score" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Elo", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "3609a3d6-a8ff-470d-8333-20744922db53", + "x": 4, + "y": 4, + "w": 2, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "Top Models By Intelligence Index", + "description": "By Index Score (Artificial Analysis)", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "source": "artificial-analysis" + }, + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "benchmark", + "operation": "equals", + "value": "Intelligence Index" + }, + { + "column": "metric", + "operation": "equals", + "value": "index" + } + ] + }, + "sort": { + "by": [ + [ + "score", + "desc" + ] + ], + "top": 15 + } + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "modelName", + "yAxisData": [ + "score" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Intelligence Index", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "bdbfd6a7-4bbe-4eab-91b5-6908a10344d7", + "x": 0, + "y": 7, + "w": 3, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "Most Used Models On OpenRouter", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[rankings]}}", + "name": "rankings", + "pluginConfigId": "{{configId}}", + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "model_permaslug", + "operation": "notequals", + "value": "other" + } + ] + }, + "group": { + "by": [ + [ + "model_permaslug", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "total_tokens" + ] + } + ] + }, + "sort": { + "by": [ + [ + "total_tokens_sum", + "desc" + ] + ], + "top": 10 + }, + "metadata": [ + { + "name": "total_tokens_sum", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-bar-chart", + "config": { + "data-stream-bar-chart": { + "xAxisData": "model_permaslug_uniqueValues", + "yAxisData": [ + "total_tokens_sum" + ], + "xAxisGroup": "none", + "xAxisLabel": "Model", + "yAxisLabel": "Tokens", + "showXAxisLabel": true, + "showYAxisLabel": true, + "showLegend": false, + "legendPosition": "bottom", + "showGrid": true, + "horizontalLayout": "horizontal", + "displayMode": "actual", + "showTotals": false, + "showValue": false, + "grouping": false, + "range": { + "type": "auto" + } + } + } + } + } + }, + { + "i": "22bac160-9e66-4597-b230-270811258056", + "x": 3, + "y": 7, + "w": 3, + "h": 3, + "config": { + "_type": "tile/data-stream", + "title": "OpenRouter Token Share By Author", + "description": "", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[rankings]}}", + "name": "rankings", + "pluginConfigId": "{{configId}}", + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "model_permaslug", + "operation": "notequals", + "value": "other" + } + ] + }, + "group": { + "by": [ + [ + "author", + "uniqueValues" + ] + ], + "aggregate": [ + { + "type": "sum", + "names": [ + "total_tokens" + ] + } + ] + }, + "sort": { + "by": [ + [ + "total_tokens_sum", + "desc" + ] + ] + }, + "metadata": [ + { + "name": "total_tokens_sum", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "pattern": ".*" + } + ] + }, + "visualisation": { + "type": "data-stream-donut-chart", + "config": { + "data-stream-donut-chart": { + "valueColumn": "total_tokens_sum", + "labelColumn": "author_uniqueValues", + "hideCenterValue": false, + "showValuesAsPercentage": true, + "legendPosition": "auto", + "legendMode": "table" + } + } + } + } + }, + { + "i": "071c926f-0d3c-4192-962f-d191aa03dcbc", + "x": 0, + "y": 10, + "w": 6, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Top Models On OpenRouter By Week (This Year)", + "description": "Each week's 20 most used models across OpenRouter, by tokens. All other models in that week are combined into Other.", + "timeframe": "thisYear", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[rankings]}}", + "name": "rankings", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "period": "week", + "top": 20 + }, + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Week" + }, + "y": { + "field": "total_tokens", + "type": "quantitative", + "aggregate": "sum", + "title": "Tokens" + }, + "series": { + "field": "model_permaslug", + "type": "nominal", + "title": "Model" + } + }, + "style": { + "cartesianStack": "on", + "legendPosition": "bottom" + } + } + }, + "visualisation": { + "type": "data-stream-line-graph", + "config": { + "data-stream-line-graph": { + "xAxisColumn": "date", + "yAxisColumn": [ + "total_tokens" + ], + "seriesColumn": "model_permaslug", + "showLegend": true, + "legendPosition": "bottom", + "yAxisLabel": "Tokens", + "showYAxisLabel": true, + "showTrendLine": false + } + } + } + } + }, + { + "i": "a3384d79-77a8-49c0-bb08-62d6744303e1", + "x": 0, + "y": 14, + "w": 6, + "h": 4, + "config": { + "_type": "tile/data-stream", + "title": "Top Model By Task", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[taskClassifications]}}", + "name": "taskClassifications", + "pluginConfigId": "{{configId}}", + "sort": { + "by": [ + [ + "usageShare", + "desc" + ] + ] + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "task", + "category", + "usageShare", + "topModel", + "topModelUsageShare", + "asOf" + ], + "hiddenColumns": [ + "tag", + "tokenShare", + "citation" + ] + } + } + } + } + }, + { + "i": "37f6bcdf-02a2-4c2a-861d-c9515d1c2092", + "x": 0, + "y": 18, + "w": 6, + "h": 8, + "config": { + "_type": "tile/data-stream", + "title": "Claude Code Cost Per Session", + "description": "Median cost of one Claude Code session routed through OpenRouter, by model and session length. Users can pick any model on OpenRouter, so many aren't Anthropic's.", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[sessionCost]}}", + "name": "sessionCost", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "harness": "claude-code" + }, + "sort": { + "by": [ + [ + "sessionLengthOrder", + "asc" + ], + [ + "median_session_cost_usd", + "asc" + ] + ] + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "scatter", + "encoding": { + "x": { + "field": "model_permaslug", + "type": "nominal", + "title": "Model" + }, + "y": { + "field": "median_session_cost_usd", + "type": "quantitative", + "title": "Median Session Cost ($)" + }, + "color": { + "field": "sessionLength", + "type": "nominal", + "title": "Session Length" + } + }, + "style": { + "legendPosition": "bottom", + "cartesianYAxisScale": "log", + "cartesianXAxisLabelRotation": 45, + "cartesianXAxisTickDensity": "dense", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Median Session Cost ($, log scale)", + "cartesianShowVerticalGridLines": true + } + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "model_permaslug", + "sessionLength", + "median_session_cost_usd", + "app_name" + ], + "hiddenColumns": [ + "app_slug", + "turn_range", + "sessionLengthOrder", + "asOf", + "citation" + ] + } + } + } + } + }, + { + "i": "56d4d9f8-9cc4-4852-9306-2997d9e87000", + "x": 0, + "y": 26, + "w": 6, + "h": 8, + "config": { + "_type": "tile/data-stream", + "title": "Top Agentic Models Compared", + "description": "Higher is better on every axis. Affordability scores price per 1M tokens on a log scale, from 100 at $0.01 or less to 0 at $1,000 or more. Change the ceiling with the Affordability ceiling parameter.", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}", + "filter": { + "multiOperation": "and", + "filters": [ + { + "column": "benchmarks.artificial_analysis.agentic_index", + "operation": "greaterthan", + "value": "0" + }, + { + "column": "benchmarks.artificial_analysis.intelligence_index", + "operation": "greaterthan", + "value": "0" + }, + { + "column": "benchmarks.artificial_analysis.coding_index", + "operation": "greaterthan", + "value": "0" + }, + { + "column": "name", + "operation": "notcontains", + "value": "(batch)" + }, + { + "column": "name", + "operation": "notcontains", + "value": "(free)" + } + ] + }, + "sort": { + "by": [ + [ + "benchmarks.artificial_analysis.agentic_index", + "desc" + ], + [ + "benchmarks.artificial_analysis.intelligence_index", + "desc" + ], + [ + "benchmarks.artificial_analysis.coding_index", + "desc" + ] + ], + "top": 10 + } + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "radar", + "encoding": { + "series": { + "field": "name", + "type": "nominal", + "title": "Model" + }, + "axes": [ + { + "field": "benchmarks.artificial_analysis.intelligence_index", + "type": "quantitative", + "title": "Intelligence Index" + }, + { + "field": "benchmarks.artificial_analysis.coding_index", + "type": "quantitative", + "title": "Coding Index" + }, + { + "field": "benchmarks.artificial_analysis.agentic_index", + "type": "quantitative", + "title": "Agentic Index" + }, + { + "field": "inputAffordability", + "type": "quantitative", + "title": "Input Affordability" + }, + { + "field": "outputAffordability", + "type": "quantitative", + "title": "Output Affordability" + }, + { + "field": "cacheReadAffordability", + "type": "quantitative", + "title": "Cache Read Affordability" + } + ] + }, + "style": { + "legendPosition": "right", + "radarShape": "polygon", + "radarFill": false, + "radarRadius": 80 + } + } + }, + "visualisation": { + "type": "data-stream-table", + "config": { + "data-stream-table": { + "columnOrder": [ + "name", + "author", + "benchmarks.artificial_analysis.intelligence_index", + "benchmarks.artificial_analysis.coding_index", + "benchmarks.artificial_analysis.agentic_index", + "inputPrice", + "outputPrice", + "cacheReadPrice" + ], + "hiddenColumns": [ + "id", + "canonical_slug", + "cacheReadPrice", + "top_provider.max_completion_tokens", + "inputModalities", + "outputModalities", + "knowledge_cutoff", + "created", + "expiration_date", + "pricing.prompt", + "pricing.completion", + "pricing.input_cache_read", + "inputAffordability", + "outputAffordability", + "cacheReadAffordability", + "architecture.input_modalities", + "architecture.output_modalities" + ] + } + } + } + } + }, + { + "i": "679f0166-77c4-47c1-9ae5-7e4c1721f718", + "x": 0, + "y": 34, + "w": 6, + "h": 1, + "config": { + "_type": "tile/text", + "title": "", + "description": "", + "visualisation": { + "config": { + "content": "Benchmarks: Artificial Analysis ([artificialanalysis.ai](https://artificialanalysis.ai)), Design Arena ([www.designarena.ai](https://www.designarena.ai)), and OpenRouter evals ([openrouter.ai](https://openrouter.ai)), via OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). \nRankings, tasks, and session cost: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). Task data is as of the **As Of** date in the Top Model By Task table; session cost is refreshed weekly. \nLicensed under CC BY 4.0.", + "fontSize": 12, + "align": "left", + "autoSize": true + } + } + } + } + ] + } +} diff --git a/plugins/OpenRouter/v1/defaultContent/scopes.json b/plugins/OpenRouter/v1/defaultContent/scopes.json index 4a701224..0d185f38 100644 --- a/plugins/OpenRouter/v1/defaultContent/scopes.json +++ b/plugins/OpenRouter/v1/defaultContent/scopes.json @@ -16,6 +16,23 @@ "allowMultipleSelection": false } }, + { + "name": "Model Comparison", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Model" + ] + } + }, + "variable": { + "name": "Compared Models", + "type": "object", + "default": "none", + "allowMultipleSelection": true + } + }, { "name": "Providers", "matches": { diff --git a/plugins/OpenRouter/v1/docs/README.md b/plugins/OpenRouter/v1/docs/README.md index e36d98fd..32418305 100644 --- a/plugins/OpenRouter/v1/docs/README.md +++ b/plugins/OpenRouter/v1/docs/README.md @@ -29,12 +29,12 @@ On save, the plugin checks the key in two steps. An invalid, disabled, or expire - **Spend and usage** — spend, requests, tokens, latency, and throughput over time, broken down by model, provider, workspace, API key, or app. - **API keys** — spend today, this week, and this month for each key, against its credit limit, plus an expiry warning. - **Account credits** — credits purchased, used, and remaining. -- **Model comparison** — pricing, context length, modalities, and Artificial Analysis benchmark scores for every model on OpenRouter. +- **Models** — pricing, context length, modalities, and Artificial Analysis benchmark scores for every model on OpenRouter. - **Providers** — every provider serving models on OpenRouter, with headquarters and datacenter locations. - **Provider performance** — current status, price, uptime, latency, and throughput for each provider serving a model. - **Benchmarks and rankings** — scores from Artificial Analysis, Design Arena, and OpenRouter's own evals, OpenRouter's public leaderboard of the most used models, what share of OpenRouter traffic goes to each kind of task, what a coding-agent session typically costs, and which public apps use OpenRouter most. -The out-of-the-box dashboards include an account **Overview**; **Model Comparison** and **Ecosystem** dashboards covering models, apps, task types, and providers across all of OpenRouter, which work with a standard API key; and a perspective for each **Model**, **Provider**, **Workspace**, and **API Key** (spend tiles need a management key). The Model perspective includes a current view of each provider's status, price, and throughput, and your spend and cost per 1M tokens on each provider. +The out-of-the-box dashboards include an account **Overview**; **Model Leaderboards** and **Ecosystem** dashboards covering models, apps, task types, and providers across all of OpenRouter, which work with a standard API key; a **Model Comparison** dashboard that compares any models you pick side by side on price, benchmarks, providers, session cost, and your own usage (the usage tiles need a management key); and a perspective for each **Model**, **Provider**, **Workspace**, and **API Key** (spend tiles need a management key). The Model perspective includes a current view of each provider's status, price, and throughput, and your spend and cost per 1M tokens on each provider. ## Data streams @@ -68,7 +68,7 @@ With a standard API key, only models and providers are imported; the workspace a ## Known limitations - **Benchmarks, rankings, app rankings, task classifications, and session costs refresh at most once a day** — OpenRouter limits these endpoints to 500 requests a day per account, so the plugin caches them for 24 hours. -- **Rankings licence** — rankings, app ranking, task classification, session cost, and benchmark data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Each row of these streams has a hidden **Citation** column with the exact attribution OpenRouter requires, and a hidden **As Of** date. The **Model Comparison**, **Ecosystem**, and **Model** dashboards credit the data already; show the citation on any dashboard you build from it. +- **Rankings licence** — rankings, app ranking, task classification, session cost, and benchmark data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Each row of these streams has a hidden **Citation** column with the exact attribution OpenRouter requires, and a hidden **As Of** date. The **Model Leaderboards**, **Model Comparison**, **Ecosystem**, and **Model** dashboards credit the data already; show the citation on any dashboard you build from it. - **Not every model has benchmark scores** — Artificial Analysis scores cover fewer than half of models; the rest show blank. - **Usage history** can be queried up to a year back. Latency, time to first token, throughput, and grouping by **Provider**, **Origin**, **Country**, **Finish reason**, or **Streamed** are limited to 31 days; longer timeframes return an error. - **Usage timeframes start at 12 hours** — data is bucketed by hour, day, week, or month to suit the range; periods with no activity are left out rather than shown as zero. From 067e06f2d0c6f639a6311c591126a301b50fed0c Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 14:13:12 +0100 Subject: [PATCH 07/16] Use smartviz tiles across the OpenRouter dashboards Squashed from: - Use a smartviz bar chart for Top Apps on the OpenRouter Ecosystem dashboard - Use smartviz tiles on the OpenRouter API Key dashboard - Use smartviz tiles on the OpenRouter Provider dashboard - Use smartviz tiles on the OpenRouter Workspace dashboard - Fix the Disabled tick colours on the OpenRouter API Key dashboard - Use smartviz tiles on the OpenRouter Overview dashboard Co-Authored-By: Claude Opus 5.5 --- .../v1/defaultContent/apiKey.dash.json | 124 ++++++ .../v1/defaultContent/ecosystem.dash.json | 23 ++ .../v1/defaultContent/overview.dash.json | 353 ++++++++++++++++-- .../v1/defaultContent/provider.dash.json | 153 ++++++++ .../v1/defaultContent/workspace.dash.json | 275 ++++++++++++++ 5 files changed, 890 insertions(+), 38 deletions(-) diff --git a/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json index b25cbd10..de1ab824 100644 --- a/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json @@ -60,6 +60,63 @@ "variables": [ "{{variables.[API Key]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "name", + "title": "Name", + "renderer": { + "type": "text" + } + }, + { + "field": "workspace_id", + "title": "Workspace ID", + "renderer": { + "type": "text" + } + }, + { + "field": "disabled", + "title": "Disabled", + "renderer": { + "type": "boolean", + "trueColor": "var(--statusErrorPrimary)", + "falseColor": "var(--textSecondary)" + } + }, + { + "field": "created_at", + "title": "Created", + "renderer": { + "type": "date" + } + }, + { + "field": "expires_at", + "title": "Expires", + "renderer": { + "type": "date" + } + }, + { + "field": "expiryStatus", + "title": "Expiry Status", + "renderer": { + "type": "status" + } + } + ] + }, + "style": { + "richTableTranspose": true + } + } + }, "visualisation": { "type": "data-stream-table", "config": { @@ -143,6 +200,28 @@ "variables": [ "{{variables.[API Key]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Spend ($)" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -217,6 +296,28 @@ "variables": [ "{{variables.[API Key]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Requests" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Requests" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -309,6 +410,29 @@ "variables": [ "{{variables.[API Key]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "group_uniqueValues", + "type": "nominal", + "title": "Model" + }, + "y": { + "field": "value_sum", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "cartesianHorizontal": true, + "cartesianShowXAxisTitle": true, + "cartesianShowYAxisTitle": true + } + } + }, "visualisation": { "type": "data-stream-bar-chart", "config": { diff --git a/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json b/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json index 392bb014..0ef1a0ad 100644 --- a/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/ecosystem.dash.json @@ -117,6 +117,29 @@ "top": 10 } }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "app_name", + "type": "nominal", + "title": "App" + }, + "y": { + "field": "total_tokens", + "type": "quantitative", + "title": "Tokens" + } + }, + "style": { + "cartesianHorizontal": true, + "cartesianShowXAxisTitle": true, + "cartesianShowYAxisTitle": true + } + } + }, "visualisation": { "type": "data-stream-bar-chart", "config": { diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json index 70a25672..104f8b09 100644 --- a/plugins/OpenRouter/v1/defaultContent/overview.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -11,7 +11,7 @@ "i": "bb7e5c59-ac7d-4c28-9966-a72354c327b2", "x": 0, "y": 0, - "w": 1, + "w": 2, "h": 2, "moved": false, "static": false, @@ -29,47 +29,55 @@ "name": "credits", "pluginConfigId": "{{configId}}" }, - "visualisation": { - "type": "data-stream-scalar", - "config": { - "data-stream-scalar": { - "value": "remaining", - "comparisonColumn": "none", - "label": "Credits Remaining" + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "gauge", + "encoding": { + "value": { + "field": "remaining", + "type": "quantitative", + "title": "Credits Remaining" + }, + "max": { + "field": "total_credits", + "type": "quantitative", + "title": "Total Credits" + } + }, + "style": { + "gaugeLabelText": "Credits Remaining", + "gaugeAxisLabels": true, + "gaugeBands": [ + { + "to": 0.1, + "color": "#ee6666" + }, + { + "to": 0.25, + "color": "#fac858" + }, + { + "to": 1, + "color": "#91cc75" + } + ] } } - } - } - }, - { - "i": "5f747a12-0c82-45c6-932d-734b1c4ac88d", - "x": 1, - "y": 0, - "w": 1, - "h": 2, - "moved": false, - "static": false, - "z": 0, - "config": { - "_type": "tile/data-stream", - "title": "", - "description": "", - "timeframe": "none", - "activePluginConfigIds": [ - "{{configId}}" - ], - "dataStream": { - "id": "{{dataStreams.[credits]}}", - "name": "credits", - "pluginConfigId": "{{configId}}" }, "visualisation": { - "type": "data-stream-scalar", + "type": "data-stream-gauge", "config": { - "data-stream-scalar": { - "value": "total_usage", - "comparisonColumn": "none", - "label": "Credits Used" + "data-stream-gauge": { + "value": { + "type": "mean", + "columns": [ + "remaining" + ] + }, + "label": "Credits Remaining", + "minimum": 0, + "maximumColumn": "total_credits" } } } @@ -126,6 +134,24 @@ } ] }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "scalar", + "encoding": { + "values": [ + { + "field": "value_sum", + "type": "quantitative", + "title": "Spend (This Month)" + } + ] + }, + "style": { + "scalarLabelText": "Spend (This Month)" + } + } + }, "visualisation": { "type": "data-stream-scalar", "config": { @@ -189,6 +215,24 @@ } ] }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "scalar", + "encoding": { + "values": [ + { + "field": "value_sum", + "type": "quantitative", + "title": "Requests (This Month)" + } + ] + }, + "style": { + "scalarLabelText": "Requests (This Month)" + } + } + }, "visualisation": { "type": "data-stream-scalar", "config": { @@ -248,6 +292,28 @@ } ] }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Spend ($)" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -314,6 +380,28 @@ } ] }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Requests" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Requests" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -536,6 +624,66 @@ } ] }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "name", + "title": "Model", + "renderer": { + "type": "text" + } + }, + { + "field": "spend", + "title": "Spend", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "requests", + "title": "Requests", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "costPerRequest", + "title": "Cost Per Request", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "promptTokens", + "title": "Prompt Tokens", + "align": "right", + "renderer": { + "type": "number", + "compact": true + } + }, + { + "field": "completionTokens", + "title": "Completion Tokens", + "align": "right", + "renderer": { + "type": "number", + "compact": true + } + } + ] + }, + "style": {} + } + }, "visualisation": { "type": "data-stream-table", "config": { @@ -623,6 +771,29 @@ } ] }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "group_uniqueValues", + "type": "nominal", + "title": "Model" + }, + "y": { + "field": "value_sum", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "cartesianHorizontal": true, + "cartesianShowXAxisTitle": true, + "cartesianShowYAxisTitle": true + } + } + }, "visualisation": { "type": "data-stream-bar-chart", "config": { @@ -717,6 +888,29 @@ } ] }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "pie", + "encoding": { + "label": { + "field": "group_uniqueValues", + "type": "nominal", + "title": "Workspace" + }, + "value": { + "field": "value_sum", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "pieInnerRadius": 55, + "pieDonutCenterValue": "total", + "pieLegendStyle": "table" + } + } + }, "visualisation": { "type": "data-stream-donut-chart", "config": { @@ -815,7 +1009,8 @@ }, "style": { "pieInnerRadius": 55, - "legendPosition": "right" + "pieDonutCenterValue": "total", + "pieLegendStyle": "table" } } }, @@ -957,6 +1152,88 @@ "scope": "{{scopes.[Workspaces]}}", "workspace": "{{workspaceId}}" }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "name", + "title": "Name", + "renderer": { + "type": "text" + } + }, + { + "field": "usage_monthly", + "title": "Spend This Month", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "usage_weekly", + "title": "Spend This Week", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "usage_daily", + "title": "Spend Today", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "usage", + "title": "Total Spend", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "limit", + "title": "Limit", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "limit_remaining", + "title": "Limit Remaining", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "limit_reset", + "title": "Limit Reset", + "renderer": { + "type": "text" + } + }, + { + "field": "disabled", + "title": "Disabled", + "renderer": { + "type": "boolean", + "trueColor": "var(--statusErrorPrimary)", + "falseColor": "var(--textSecondary)" + } + } + ] + }, + "style": {} + } + }, "visualisation": { "type": "data-stream-table", "config": { diff --git a/plugins/OpenRouter/v1/defaultContent/provider.dash.json b/plugins/OpenRouter/v1/defaultContent/provider.dash.json index dcae404b..4242661b 100644 --- a/plugins/OpenRouter/v1/defaultContent/provider.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/provider.dash.json @@ -36,6 +36,48 @@ "variables": [ "{{variables.[Provider]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "name", + "title": "Name", + "renderer": { + "type": "text" + } + }, + { + "field": "headquarters", + "title": "Headquarters", + "renderer": { + "type": "text" + } + }, + { + "field": "datacenters", + "title": "Datacenters", + "renderer": { + "type": "tag-list", + "separator": ", " + } + }, + { + "field": "status_page_url", + "title": "Status Page", + "renderer": { + "type": "link" + } + } + ] + }, + "style": { + "richTableTranspose": true + } + } + }, "visualisation": { "type": "data-stream-table", "config": { @@ -117,6 +159,28 @@ "variables": [ "{{variables.[Provider]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Spend ($)" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -192,6 +256,28 @@ "variables": [ "{{variables.[Provider]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Requests" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Requests" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -261,6 +347,28 @@ "variables": [ "{{variables.[Provider]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Latency p50" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Latency p50" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -335,6 +443,28 @@ "variables": [ "{{variables.[Provider]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Tokens/s" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Tokens/s" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -428,6 +558,29 @@ "variables": [ "{{variables.[Provider]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "group_uniqueValues", + "type": "nominal", + "title": "Model" + }, + "y": { + "field": "value_sum", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "cartesianHorizontal": true, + "cartesianShowXAxisTitle": true, + "cartesianShowYAxisTitle": true + } + } + }, "visualisation": { "type": "data-stream-bar-chart", "config": { diff --git a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json index 7d5f8aa8..ff3b4fe0 100644 --- a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json @@ -42,6 +42,109 @@ "variables": [ "{{variables.[Workspace]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "name", + "title": "Name", + "renderer": { + "type": "text" + } + }, + { + "field": "description", + "title": "Description", + "renderer": { + "type": "text" + } + }, + { + "field": "default_text_model", + "title": "Text Model", + "renderer": { + "type": "text" + } + }, + { + "field": "default_image_model", + "title": "Image Model", + "renderer": { + "type": "text" + } + }, + { + "field": "default_provider_sort", + "title": "Provider Sort", + "renderer": { + "type": "text" + } + }, + { + "field": "disabled_server_tools", + "title": "Disabled Tools", + "renderer": { + "type": "tag-list", + "separator": ", " + } + }, + { + "field": "include_byok_in_budgets", + "title": "BYOK In Budgets", + "renderer": { + "type": "boolean", + "falseColor": "var(--textSecondary)" + } + }, + { + "field": "is_observability_io_logging_enabled", + "title": "Private Logging", + "renderer": { + "type": "boolean", + "falseColor": "var(--textSecondary)" + } + }, + { + "field": "io_logging_sampling_rate", + "title": "Log Sample Rate", + "renderer": { + "type": "text" + } + }, + { + "field": "is_observability_broadcast_enabled", + "title": "Broadcast", + "renderer": { + "type": "boolean", + "falseColor": "var(--textSecondary)" + } + }, + { + "field": "is_data_discount_logging_enabled", + "title": "Discount Logging", + "renderer": { + "type": "boolean", + "falseColor": "var(--textSecondary)" + } + }, + { + "field": "created_at", + "title": "Created", + "renderer": { + "type": "date" + } + } + ] + }, + "style": { + "richTableTranspose": true, + "richTableFontSize": 12 + } + } + }, "visualisation": { "type": "data-stream-table", "config": { @@ -126,6 +229,28 @@ "variables": [ "{{variables.[Workspace]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Spend ($)" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -200,6 +325,28 @@ "variables": [ "{{variables.[Workspace]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Requests" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Requests" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -292,6 +439,29 @@ "variables": [ "{{variables.[Workspace]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "group_uniqueValues", + "type": "nominal", + "title": "Model" + }, + "y": { + "field": "value_sum", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "cartesianHorizontal": true, + "cartesianShowXAxisTitle": true, + "cartesianShowYAxisTitle": true + } + } + }, "visualisation": { "type": "data-stream-bar-chart", "config": { @@ -394,6 +564,29 @@ "variables": [ "{{variables.[Workspace]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "pie", + "encoding": { + "label": { + "field": "group_uniqueValues", + "type": "nominal", + "title": "API Key" + }, + "value": { + "field": "value_sum", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "pieInnerRadius": 55, + "pieDonutCenterValue": "total", + "pieLegendStyle": "table" + } + } + }, "visualisation": { "type": "data-stream-donut-chart", "config": { @@ -447,6 +640,88 @@ "variables": [ "{{variables.[Workspace]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "name", + "title": "Name", + "renderer": { + "type": "text" + } + }, + { + "field": "usage_monthly", + "title": "Spend This Month", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "usage_weekly", + "title": "Spend This Week", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "usage_daily", + "title": "Spend Today", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "usage", + "title": "Total Spend", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "limit", + "title": "Limit", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "limit_remaining", + "title": "Limit Remaining", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "limit_reset", + "title": "Limit Reset", + "renderer": { + "type": "text" + } + }, + { + "field": "disabled", + "title": "Disabled", + "renderer": { + "type": "boolean", + "trueColor": "var(--statusErrorPrimary)", + "falseColor": "var(--textSecondary)" + } + } + ] + }, + "style": {} + } + }, "visualisation": { "type": "data-stream-table", "config": { From 56202201f1cfc028e88ce051bbe08d5fa2147967 Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 15:57:19 +0100 Subject: [PATCH 08/16] Extend OpenRouter Model Leaderboards and fix Model drilldown links Squashed from: - Look up models from the models stream on the OpenRouter Model Usage tile - Add a month-to-date leaderboard to OpenRouter Model Leaderboards - Show This Month on the top row of OpenRouter Model Leaderboards - Add benchmark and task leader tiles to OpenRouter Model Leaderboards - Finish the smartviz pass on OpenRouter Model Leaderboards - Add a Model page link to the OpenRouter benchmarks scatter's data - Fix Model drilldown links on OpenRouter Model Comparison and Overview - Use smartviz tiles on the OpenRouter Model dashboard Co-Authored-By: Claude Opus 5.5 --- .../v1/defaultContent/model.dash.json | 551 ++++++++- .../defaultContent/modelComparison.dash.json | 48 +- .../modelLeaderboards.dash.json | 1086 ++++++++++++----- .../v1/defaultContent/overview.dash.json | 33 +- 4 files changed, 1362 insertions(+), 356 deletions(-) diff --git a/plugins/OpenRouter/v1/defaultContent/model.dash.json b/plugins/OpenRouter/v1/defaultContent/model.dash.json index ec971f8c..965bdb31 100644 --- a/plugins/OpenRouter/v1/defaultContent/model.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/model.dash.json @@ -36,6 +36,49 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "name", + "title": "Name", + "renderer": { + "type": "text" + } + }, + { + "field": "author", + "title": "Author", + "renderer": { + "type": "text" + } + }, + { + "field": "canonical_slug", + "title": "Slug", + "renderer": { + "type": "text" + } + }, + { + "field": "context_length", + "title": "Context", + "renderer": { + "type": "number", + "compact": true + } + } + ] + }, + "style": { + "richTableTranspose": true, + "richTableFontSize": 12 + } + } + }, "visualisation": { "type": "data-stream-table", "config": { @@ -94,6 +137,110 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "inputPrice", + "title": "Input $/1M", + "renderer": { + "type": "text" + } + }, + { + "field": "outputPrice", + "title": "Output $/1M", + "renderer": { + "type": "text" + } + }, + { + "field": "cacheReadPrice", + "title": "Cache Read $/1M", + "renderer": { + "type": "text" + } + }, + { + "field": "context_length", + "title": "Context", + "renderer": { + "type": "number", + "compact": true + } + }, + { + "field": "top_provider.max_completion_tokens", + "title": "Max Output", + "renderer": { + "type": "number", + "compact": true + } + }, + { + "field": "inputModalities", + "title": "Input", + "renderer": { + "type": "tag-list" + } + }, + { + "field": "outputModalities", + "title": "Output", + "renderer": { + "type": "tag-list" + } + }, + { + "field": "benchmarks.artificial_analysis.intelligence_index", + "title": "Intelligence", + "renderer": { + "type": "number", + "decimals": 1 + } + }, + { + "field": "benchmarks.artificial_analysis.coding_index", + "title": "Coding", + "renderer": { + "type": "number", + "decimals": 1 + } + }, + { + "field": "benchmarks.artificial_analysis.agentic_index", + "title": "Agentic", + "renderer": { + "type": "number", + "decimals": 1 + } + }, + { + "field": "knowledge_cutoff", + "title": "Knowledge Cutoff", + "renderer": { + "type": "date" + } + }, + { + "field": "expiration_date", + "title": "Expires", + "renderer": { + "type": "date" + } + } + ] + }, + "style": { + "richTableTranspose": true, + "richTableFontSize": 12, + "richTableNoHeader": true + } + } + }, "visualisation": { "type": "data-stream-table", "config": { @@ -188,6 +335,28 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Spend ($)" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Spend ($)" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -262,6 +431,28 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Requests" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Requests" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -502,6 +693,34 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Spend ($)" + }, + "series": { + "field": "group", + "type": "nominal", + "title": "Provider" + } + }, + "style": { + "legendPosition": "bottom", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Spend ($)" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -578,6 +797,34 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Cost per 1M tokens ($)" + }, + "series": { + "field": "group", + "type": "nominal", + "title": "Provider" + } + }, + "style": { + "legendPosition": "bottom", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Cost per 1M tokens ($)" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -647,6 +894,28 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Latency p50" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Latency p50" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -716,6 +985,28 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "line", + "encoding": { + "x": { + "field": "date", + "type": "temporal", + "title": "Date" + }, + "y": { + "field": "value", + "type": "quantitative", + "title": "Time To First Token p50" + } + }, + "style": { + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Time To First Token p50" + } + } + }, "visualisation": { "type": "data-stream-line-graph", "config": { @@ -774,6 +1065,143 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "provider_name", + "title": "Provider", + "renderer": { + "type": "text" + } + }, + { + "field": "health", + "title": "Status", + "renderer": { + "type": "status" + } + }, + { + "field": "tag", + "title": "Endpoint", + "renderer": { + "type": "text" + } + }, + { + "field": "quantization", + "title": "Quantization", + "renderer": { + "type": "text" + } + }, + { + "field": "context_length", + "title": "Context", + "align": "right", + "renderer": { + "type": "number", + "compact": true + } + }, + { + "field": "inputPrice", + "title": "Input $/1M", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "outputPrice", + "title": "Output $/1M", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "cacheReadPrice", + "title": "Cache Read $/1M", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "uptime_last_30m", + "title": "Uptime (30m)", + "align": "right", + "renderer": { + "type": "number", + "decimals": 1, + "suffix": "%", + "thresholds": [ + { + "to": 95, + "color": "var(--statusErrorPrimary)" + }, + { + "to": 99, + "color": "var(--statusWarningPrimary)" + }, + { + "to": 100, + "color": "var(--statusHealthyPrimary)" + } + ] + } + }, + { + "field": "uptime_last_1d", + "title": "Uptime (1d)", + "align": "right", + "renderer": { + "type": "number", + "decimals": 1, + "suffix": "%", + "thresholds": [ + { + "to": 95, + "color": "var(--statusErrorPrimary)" + }, + { + "to": 99, + "color": "var(--statusWarningPrimary)" + }, + { + "to": 100, + "color": "var(--statusHealthyPrimary)" + } + ] + } + }, + { + "field": "latency_last_30m.p50", + "title": "Latency p50", + "align": "right", + "renderer": { + "type": "text" + } + }, + { + "field": "throughput_last_30m.p50", + "title": "Tokens/s p50", + "align": "right", + "renderer": { + "type": "number", + "decimals": 1 + } + } + ] + }, + "style": {} + } + }, "visualisation": { "type": "data-stream-table", "config": { @@ -918,36 +1346,96 @@ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[modelProviders]}}", - "name": "modelProviders", - "pluginConfigId": "{{configId}}", - "sort": { - "by": [ - [ - "latency_last_30m.p50", - "asc" - ] + "id": "datastream-sql", + "dataSourceConfig": { + "version": "2.0", + "sql": "SELECT\n e.tag AS endpoint,\n p.percentile,\n CASE p.percentile WHEN 'p50' THEN e.\"latency_last_30m.p50\" ELSE e.\"latency_last_30m.p90\" END AS latency\nFROM Endpoints e, (VALUES ('p50'), ('p90')) AS p(percentile)\nORDER BY e.\"latency_last_30m.p50\" NULLS LAST, e.tag, p.percentile", + "tables": [ + { + "tableName": "Endpoints", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[modelProviders]}}", + "name": "modelProviders", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.[Models]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[Model]}}" + }, + "variables": [ + "{{variables.[Model]}}" + ], + "timeframe": "none" + } + } ] - } - }, - "scope": { - "scope": "{{scopes.[Models]}}", - "workspace": "{{workspaceId}}", - "variable": "{{variables.[Model]}}" + }, + "metadata": [ + { + "name": "endpoint", + "displayName": "Endpoint", + "shape": "string", + "role": "label" + }, + { + "name": "percentile", + "displayName": "Percentile", + "shape": "string" + }, + { + "name": "latency", + "displayName": "Latency", + "shape": "milliseconds" + } + ] }, "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "endpoint", + "type": "nominal", + "title": "Endpoint" + }, + "y": { + "field": "latency", + "type": "quantitative", + "title": "Latency" + }, + "series": { + "field": "percentile", + "type": "nominal", + "title": "Percentile" + } + }, + "style": { + "cartesianHorizontal": true, + "cartesianStack": "off", + "cartesianShowXAxisTitle": true, + "cartesianShowYAxisTitle": true, + "legendPosition": "bottom" + } + } + }, "visualisation": { "type": "data-stream-bar-chart", "config": { "data-stream-bar-chart": { - "xAxisData": "tag", + "xAxisData": "endpoint", "yAxisData": [ - "latency_last_30m.p50", - "latency_last_30m.p90" + "latency" ], - "xAxisGroup": "none", + "xAxisGroup": "percentile", "xAxisLabel": "Endpoint", "yAxisLabel": "Latency", "showXAxisLabel": true, @@ -959,7 +1447,7 @@ "displayMode": "actual", "showTotals": false, "showValue": false, - "grouping": false, + "grouping": true, "range": { "type": "auto" } @@ -1006,6 +1494,29 @@ "variables": [ "{{variables.[Model]}}" ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "bar", + "encoding": { + "x": { + "field": "tag", + "type": "nominal", + "title": "Endpoint" + }, + "y": { + "field": "throughput_last_30m.p50", + "type": "quantitative", + "title": "Tokens/s" + } + }, + "style": { + "cartesianHorizontal": true, + "cartesianShowXAxisTitle": true, + "cartesianShowYAxisTitle": true + } + } + }, "visualisation": { "type": "data-stream-bar-chart", "config": { diff --git a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json index 252eb8c0..65f44e62 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json @@ -31,7 +31,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "SELECT\n s.sourceId AS id,\n m.name,\n m.author,\n m.inputPrice,\n m.outputPrice,\n m.cacheReadPrice,\n m.context_length AS contextLength,\n m.\"top_provider.max_completion_tokens\" AS maxCompletionTokens,\n m.inputModalities,\n m.outputModalities,\n m.\"benchmarks.artificial_analysis.intelligence_index\" AS intelligenceIndex,\n m.\"benchmarks.artificial_analysis.coding_index\" AS codingIndex,\n m.\"benchmarks.artificial_analysis.agentic_index\" AS agenticIndex,\n m.knowledge_cutoff AS knowledgeCutoff,\n m.created,\n m.expiration_date AS expirationDate\nFROM Models m\nJOIN Selected s ON m.id = s.rawId\nORDER BY m.name", + "sql": "SELECT\n s.rawId AS id,\n CASE WHEN s.rawId IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(s.rawId, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n m.name,\n m.author,\n m.inputPrice,\n m.outputPrice,\n m.cacheReadPrice,\n m.context_length AS contextLength,\n m.\"top_provider.max_completion_tokens\" AS maxCompletionTokens,\n m.inputModalities,\n m.outputModalities,\n m.\"benchmarks.artificial_analysis.intelligence_index\" AS intelligenceIndex,\n m.\"benchmarks.artificial_analysis.coding_index\" AS codingIndex,\n m.\"benchmarks.artificial_analysis.agentic_index\" AS agenticIndex,\n m.knowledge_cutoff AS knowledgeCutoff,\n m.created,\n m.expiration_date AS expirationDate\nFROM Models m\nJOIN Selected s ON m.id = s.rawId\nORDER BY m.name", "tables": [ { "tableName": "Models", @@ -74,17 +74,18 @@ "shape": "string", "visible": false }, + { + "name": "url", + "displayName": "Model Link", + "shape": "string", + "visible": false + }, { "name": "name", "displayName": "Model", "shape": "string", "role": "label" }, - { - "sourceId": "id", - "sourceType": "Model", - "name": "name" - }, { "name": "author", "displayName": "Author", @@ -216,9 +217,9 @@ "expirationDate" ], "hiddenColumns": [ - "id" - ], - "rowLinkColumnName": "name" + "id", + "url" + ] } } }, @@ -229,10 +230,11 @@ "encoding": { "columns": [ { - "field": "name", + "field": "url", "title": "Model", "renderer": { - "type": "text" + "type": "link", + "labelField": "name" } }, { @@ -949,7 +951,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "WITH by_slug AS (\n SELECT\n canonical_slug,\n sourceId,\n name\n FROM Selected\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n s.sourceId AS id,\n s.name,\n b.benchmark,\n b.metric,\n b.score,\n b.tasks,\n b.avgCostPerTask,\n b.avgDurationMs\nFROM Benchmarks b\nJOIN by_slug s ON b.model = s.canonical_slug\nWHERE b.source = 'openrouter'\nORDER BY b.score DESC", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug,\n sourceId,\n rawId,\n name\n FROM Selected\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n s.rawId AS id,\n CASE WHEN s.rawId IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(s.rawId, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n s.name,\n b.benchmark,\n b.metric,\n b.score,\n b.tasks,\n b.avgCostPerTask,\n b.avgDurationMs\nFROM Benchmarks b\nJOIN by_slug s ON b.model = s.canonical_slug\nWHERE b.source = 'openrouter'\nORDER BY b.score DESC", "tables": [ { "tableName": "Benchmarks", @@ -992,17 +994,18 @@ "shape": "string", "visible": false }, + { + "name": "url", + "displayName": "Model Link", + "shape": "string", + "visible": false + }, { "name": "name", "displayName": "Model", "shape": "string", "role": "label" }, - { - "sourceId": "id", - "sourceType": "Model", - "name": "name" - }, { "name": "benchmark", "displayName": "Benchmark", @@ -1058,9 +1061,9 @@ "avgDurationMs" ], "hiddenColumns": [ - "id" - ], - "rowLinkColumnName": "name" + "id", + "url" + ] } } }, @@ -1071,10 +1074,11 @@ "encoding": { "columns": [ { - "field": "name", + "field": "url", "title": "Model", "renderer": { - "type": "text" + "type": "link", + "labelField": "name" } }, { diff --git a/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json b/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json index 5f8c783b..0c82a44f 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json @@ -8,30 +8,222 @@ "version": 1, "contents": [ { - "i": "0c29cf9b-46f4-478d-98c4-25a07523750b", + "i": "0c4d2c18-8f65-42e2-9a44-cdd6c0efe180", "x": 0, "y": 0, - "w": 6, + "w": 4, "h": 4, "config": { "_type": "tile/data-stream", - "title": "Models", - "description": "", + "title": "Top Models On OpenRouter (This Month)", + "description": "Top 10 models by tokens so far this month, compared with the same days of last month. Rank change is against last month's ranking over those days.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[models]}}", - "name": "models", - "pluginConfigId": "{{configId}}", - "sort": { - "by": [ - [ - "benchmarks.artificial_analysis.intelligence_index", - "desc" - ] + "id": "datastream-sql", + "dataSourceConfig": { + "version": "2.0", + "sql": "-- Month to date vs the same days of last month: last month only counts days up to\n-- the latest day this month has data for, so a partial month compares like for like.\n-- Dates arrive as ISO strings (YYYY-MM-DD...), so month and day are sliced from the text.\n-- Timeframe edges are local time but dates are UTC, so each table can spill a day into the\n-- neighbouring month. Last month is the month most of LastMonth's rows fall in (it is never\n-- empty, unlike ThisMonth on the 1st), and this month is the one after it.\nWITH days AS (\n SELECT 'this' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM ThisMonth\n WHERE model_permaslug <> 'other'\n UNION ALL\n SELECT 'last' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM LastMonth\n WHERE model_permaslug <> 'other'\n),\nprevious AS (\n SELECT mode(substr(day, 1, 7)) AS previous_month\n FROM days\n WHERE source = 'last'\n),\nmonths AS (\n SELECT\n strftime(CAST(previous_month || '-01' AS DATE) + INTERVAL 1 MONTH, '%Y-%m') AS month,\n previous_month\n FROM previous\n),\ncutoff AS (\n SELECT\n m.month,\n m.previous_month,\n (SELECT max(CAST(substr(day, 9, 2) AS INTEGER)) FROM days WHERE source = 'this' AND substr(day, 1, 7) = m.month) AS last_day\n FROM months m\n),\nthis_month AS (\n SELECT d.slug, sum(d.total_tokens) AS tokens\n FROM days d, cutoff\n WHERE d.source = 'this' AND substr(d.day, 1, 7) = cutoff.month\n GROUP BY d.slug\n),\nlast_month AS (\n SELECT\n d.slug,\n sum(d.total_tokens) AS tokens,\n row_number() OVER (ORDER BY sum(d.total_tokens) DESC) AS rank\n FROM days d, cutoff\n WHERE d.source = 'last'\n AND substr(d.day, 1, 7) = cutoff.previous_month\n AND CAST(substr(d.day, 9, 2) AS INTEGER) <= cutoff.last_day\n GROUP BY d.slug\n),\nranked AS (\n SELECT\n slug,\n tokens,\n row_number() OVER (ORDER BY tokens DESC) AS rank\n FROM this_month\n),\nby_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n r.rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, r.slug) AS name,\n r.tokens,\n l.tokens AS lastTokens,\n (r.tokens - l.tokens) / l.tokens AS change,\n COALESCE('#' || CAST(l.rank AS VARCHAR), 'New') AS lastRank,\n l.rank - r.rank AS rankChange\nFROM ranked r\nLEFT JOIN last_month l ON l.slug = r.slug\nLEFT JOIN by_slug m ON m.slug = r.slug\nWHERE r.rank <= 10\nORDER BY r.rank", + "tables": [ + { + "tableName": "ThisMonth", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[rankings]}}", + "name": "rankings", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "period": "day" + } + }, + "timeframe": "thisMonth" + } + }, + { + "tableName": "LastMonth", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[rankings]}}", + "name": "rankings", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "period": "day" + } + }, + "timeframe": "lastMonth" + } + }, + { + "tableName": "Models", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } + } ] + }, + "metadata": [ + { + "name": "rank", + "displayName": "Rank", + "shape": "number" + }, + { + "name": "id", + "displayName": "Model ID", + "shape": "string", + "visible": false + }, + { + "name": "url", + "displayName": "Model Link", + "shape": "string", + "visible": false + }, + { + "name": "name", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "tokens", + "displayName": "Tokens (This Month)", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "name": "lastTokens", + "displayName": "Same Days Last Month", + "shape": [ + "number", + { + "decimalPlaces": 0, + "thousandsSeparator": true + } + ] + }, + { + "name": "change", + "displayName": "Token Change", + "shape": [ + "percent", + { + "asZeroToOne": true, + "decimalPlaces": 0 + } + ] + }, + { + "name": "lastRank", + "displayName": "Last Month Rank", + "shape": "string" + }, + { + "name": "rankChange", + "displayName": "Rank Change", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + } + ] + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "rank", + "title": "#", + "align": "right", + "width": 50, + "renderer": { + "type": "number", + "decimals": 0 + } + }, + { + "field": "url", + "title": "Model", + "renderer": { + "type": "link", + "labelField": "name" + } + }, + { + "field": "tokens", + "title": "Tokens (This Month)", + "align": "right", + "renderer": { + "type": "number", + "compact": true + } + }, + { + "field": "lastTokens", + "title": "Same Days Last Month", + "align": "right", + "renderer": { + "type": "number", + "compact": true + } + }, + { + "field": "change", + "title": "Token Change", + "align": "right", + "renderer": { + "type": "delta" + } + }, + { + "field": "lastRank", + "title": "Last Month Rank", + "align": "center", + "renderer": { + "type": "badge", + "colors": { + "New": "rgba(22, 163, 74, 0.55)" + }, + "defaultColor": "rgba(71, 85, 105, 0.5)" + } + }, + { + "field": "rankChange", + "title": "Rank Change", + "align": "right", + "renderer": { + "type": "delta" + } + } + ] + }, + "style": {} } }, "visualisation": { @@ -40,33 +232,17 @@ "data-stream-table": { "transpose": false, "columnOrder": [ + "rank", "name", - "author", - "inputPrice", - "outputPrice", - "context_length", - "benchmarks.artificial_analysis.intelligence_index", - "benchmarks.artificial_analysis.coding_index", - "benchmarks.artificial_analysis.agentic_index" + "tokens", + "lastTokens", + "change", + "lastRank", + "rankChange" ], "hiddenColumns": [ "id", - "canonical_slug", - "cacheReadPrice", - "top_provider.max_completion_tokens", - "inputModalities", - "outputModalities", - "knowledge_cutoff", - "created", - "expiration_date", - "pricing.prompt", - "pricing.completion", - "pricing.input_cache_read", - "inputAffordability", - "outputAffordability", - "cacheReadAffordability", - "architecture.input_modalities", - "architecture.output_modalities" + "url" ] } } @@ -74,322 +250,486 @@ } }, { - "i": "3eec66c4-de77-40ee-b403-37620ac6ec6b", + "i": "1e2875d1-b28b-4f04-ab27-47b78899619d", "x": 0, "y": 4, - "w": 2, - "h": 3, + "w": 6, + "h": 4, "config": { "_type": "tile/data-stream", - "title": "Cost Per Task For Top GPQA Models", - "description": "By Accuracy", + "title": "Intelligence Index Vs Input Price (Top 20)", + "description": "Top 20 models by Artificial Analysis Intelligence Index, against their input price. Higher and further left is better value.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[benchmarks]}}", - "name": "benchmarks", - "pluginConfigId": "{{configId}}", + "id": "datastream-sql", "dataSourceConfig": { - "source": "openrouter" - }, - "filter": { - "multiOperation": "and", - "filters": [ + "version": "2.0", + "sql": "WITH by_slug AS (\n SELECT\n id,\n name AS model,\n author,\n inputPrice,\n \"benchmarks.artificial_analysis.intelligence_index\" AS intelligence\n FROM Models\n WHERE \"benchmarks.artificial_analysis.intelligence_index\" IS NOT NULL\n AND inputPrice IS NOT NULL\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n id,\n model,\n '/drilldown/node?sourceId=Model%2F' || replace(replace(id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' AS url,\n author,\n inputPrice,\n intelligence\nFROM by_slug\nORDER BY intelligence DESC\nLIMIT 20", + "tables": [ { - "column": "benchmark", - "operation": "equals", - "value": "gpqa_diamond" - }, - { - "column": "metric", - "operation": "equals", - "value": "accuracy" + "tableName": "Models", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } } ] }, - "sort": { - "by": [ - [ - "score", - "desc" + "metadata": [ + { + "name": "id", + "displayName": "Model ID", + "shape": "string", + "visible": false + }, + { + "name": "model", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "url", + "displayName": "Link", + "shape": [ + "url", + { + "label": "Model page" + } + ] + }, + { + "name": "author", + "displayName": "Author", + "shape": "string" + }, + { + "name": "inputPrice", + "displayName": "Input Price ($/1M Tokens)", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 2 + } + ] + }, + { + "name": "intelligence", + "displayName": "Intelligence Index", + "shape": [ + "number", + { + "decimalPlaces": 1 + } ] - ], - "top": 15 - } - }, - "visualisation": { - "type": "data-stream-bar-chart", - "config": { - "data-stream-bar-chart": { - "xAxisData": "modelName", - "yAxisData": [ - "avgCostPerTask" - ], - "xAxisGroup": "none", - "xAxisLabel": "Model", - "yAxisLabel": "Avg Cost Per Task (USD)", - "showXAxisLabel": true, - "showYAxisLabel": true, - "showLegend": false, - "legendPosition": "bottom", - "showGrid": true, - "horizontalLayout": "horizontal", - "displayMode": "actual", - "showTotals": false, - "showValue": false, - "grouping": false, - "range": { - "type": "auto" - } } - } - } - } - }, - { - "i": "df0c3dda-bfa5-48a0-babf-ed96590b4be2", - "x": 2, - "y": 4, - "w": 2, - "h": 3, - "config": { - "_type": "tile/data-stream", - "title": "Top Models For Website Design (Elo)", - "description": "By Elo Rating (Design Arena)", - "timeframe": "none", - "activePluginConfigIds": [ - "{{configId}}" - ], - "dataStream": { - "id": "{{dataStreams.[benchmarks]}}", - "name": "benchmarks", - "pluginConfigId": "{{configId}}", - "dataSourceConfig": { - "source": "design-arena" - }, - "filter": { - "multiOperation": "and", - "filters": [ - { - "column": "benchmark", - "operation": "equals", - "value": "website" + ] + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "scatter", + "encoding": { + "x": { + "field": "inputPrice", + "type": "quantitative", + "title": "Input Price ($/1M Tokens)" }, - { - "column": "metric", - "operation": "equals", - "value": "elo" + "y": { + "field": "intelligence", + "type": "quantitative", + "title": "Intelligence Index" + }, + "color": { + "field": "author", + "type": "nominal", + "title": "Author" } - ] - }, - "sort": { - "by": [ - [ - "score", - "desc" - ] - ], - "top": 15 + }, + "style": { + "cartesianShowXAxisTitle": true, + "cartesianShowYAxisTitle": true, + "cartesianYAxisRangeMode": "fit", + "cartesianPointRadius": 7, + "cartesianShowVerticalGridLines": true, + "legendPosition": "bottom", + "cartesianTooltipTemplate": "{{{{raw}}}}{{model}}: {{value}}{{{{/raw}}}}" + } } }, "visualisation": { - "type": "data-stream-bar-chart", + "type": "data-stream-table", "config": { - "data-stream-bar-chart": { - "xAxisData": "modelName", - "yAxisData": [ - "score" + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "model", + "url", + "author", + "inputPrice", + "intelligence" ], - "xAxisGroup": "none", - "xAxisLabel": "Model", - "yAxisLabel": "Elo", - "showXAxisLabel": true, - "showYAxisLabel": true, - "showLegend": false, - "legendPosition": "bottom", - "showGrid": true, - "horizontalLayout": "horizontal", - "displayMode": "actual", - "showTotals": false, - "showValue": false, - "grouping": false, - "range": { - "type": "auto" - } + "hiddenColumns": [ + "id" + ] } } } } }, { - "i": "3609a3d6-a8ff-470d-8333-20744922db53", - "x": 4, - "y": 4, - "w": 2, + "i": "3eec66c4-de77-40ee-b403-37620ac6ec6b", + "x": 0, + "y": 8, + "w": 3, "h": 3, "config": { "_type": "tile/data-stream", - "title": "Top Models By Intelligence Index", - "description": "By Index Score (Artificial Analysis)", + "title": "Top GPQA Models By Accuracy And Cost", + "description": "Top 15 models by accuracy on GPQA Diamond, graduate-level science questions that can't be answered by searching.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[benchmarks]}}", - "name": "benchmarks", - "pluginConfigId": "{{configId}}", + "id": "datastream-sql", "dataSourceConfig": { - "source": "artificial-analysis" - }, - "filter": { - "multiOperation": "and", - "filters": [ + "version": "2.0", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, b.modelName) AS model,\n b.score AS accuracy,\n b.avgCostPerTask\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'gpqa_diamond' AND b.metric = 'accuracy'\nORDER BY b.score DESC\nLIMIT 15", + "tables": [ { - "column": "benchmark", - "operation": "equals", - "value": "Intelligence Index" + "tableName": "Benchmarks", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "source": "openrouter" + } + }, + "timeframe": "none" + } }, { - "column": "metric", - "operation": "equals", - "value": "index" + "tableName": "Models", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } } ] }, - "sort": { - "by": [ - [ - "score", - "desc" + "metadata": [ + { + "name": "rank", + "displayName": "Rank", + "shape": "number" + }, + { + "name": "id", + "displayName": "Model ID", + "shape": "string", + "visible": false + }, + { + "name": "url", + "displayName": "Model Link", + "shape": "string", + "visible": false + }, + { + "name": "model", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "accuracy", + "displayName": "Accuracy", + "shape": [ + "percent", + { + "asZeroToOne": true, + "decimalPlaces": 1 + } ] - ], - "top": 15 + }, + { + "name": "avgCostPerTask", + "displayName": "Avg Cost Per Task", + "shape": [ + "currency", + { + "code": "USD", + "decimalPlaces": 3 + } + ] + } + ] + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "rank", + "title": "#", + "align": "right", + "width": 40, + "renderer": { + "type": "number", + "decimals": 0 + } + }, + { + "field": "url", + "title": "Model", + "width": 240, + "renderer": { + "type": "link", + "labelField": "model" + } + }, + { + "field": "accuracy", + "title": "Accuracy", + "align": "right", + "width": 110, + "renderer": { + "type": "text" + } + }, + { + "field": "avgCostPerTask", + "title": "Avg Cost Per Task", + "renderer": { + "type": "bar" + } + } + ] + }, + "style": {} } }, "visualisation": { - "type": "data-stream-bar-chart", + "type": "data-stream-table", "config": { - "data-stream-bar-chart": { - "xAxisData": "modelName", - "yAxisData": [ - "score" + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "rank", + "model", + "accuracy", + "avgCostPerTask" ], - "xAxisGroup": "none", - "xAxisLabel": "Model", - "yAxisLabel": "Intelligence Index", - "showXAxisLabel": true, - "showYAxisLabel": true, - "showLegend": false, - "legendPosition": "bottom", - "showGrid": true, - "horizontalLayout": "horizontal", - "displayMode": "actual", - "showTotals": false, - "showValue": false, - "grouping": false, - "range": { - "type": "auto" - } + "hiddenColumns": [ + "id", + "url" + ] } } } } }, { - "i": "bdbfd6a7-4bbe-4eab-91b5-6908a10344d7", - "x": 0, - "y": 7, + "i": "df0c3dda-bfa5-48a0-babf-ed96590b4be2", + "x": 3, + "y": 8, "w": 3, "h": 3, "config": { "_type": "tile/data-stream", - "title": "Most Used Models On OpenRouter", - "description": "", + "title": "Top Models For Website Design", + "description": "Top 15 models by Elo rating on Design Arena's website arena, from head-to-head votes. Win rate is how often voters picked the model's site.", + "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[rankings]}}", - "name": "rankings", - "pluginConfigId": "{{configId}}", - "filter": { - "multiOperation": "and", - "filters": [ + "id": "datastream-sql", + "dataSourceConfig": { + "version": "2.0", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n id\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\n-- Design Arena's own names keep reasoning-effort variants apart (e.g. \"(xhigh)\", \"Max\")\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n b.modelName AS model,\n b.score AS elo,\n b.winRate,\n b.avgDurationMs\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'website' AND b.metric = 'elo'\nORDER BY b.score DESC\nLIMIT 15", + "tables": [ { - "column": "model_permaslug", - "operation": "notequals", - "value": "other" - } - ] - }, - "group": { - "by": [ - [ - "model_permaslug", - "uniqueValues" - ] - ], - "aggregate": [ + "tableName": "Benchmarks", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[benchmarks]}}", + "name": "benchmarks", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "source": "design-arena" + } + }, + "timeframe": "none" + } + }, { - "type": "sum", - "names": [ - "total_tokens" - ] + "tableName": "Models", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } } ] }, - "sort": { - "by": [ - [ - "total_tokens_sum", - "desc" - ] - ], - "top": 10 - }, "metadata": [ { - "name": "total_tokens_sum", + "name": "rank", + "displayName": "Rank", + "shape": "number" + }, + { + "name": "id", + "displayName": "Model ID", + "shape": "string", + "visible": false + }, + { + "name": "url", + "displayName": "Model Link", + "shape": "string", + "visible": false + }, + { + "name": "model", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "elo", + "displayName": "Elo", "shape": [ "number", { - "decimalPlaces": 0, - "thousandsSeparator": true + "decimalPlaces": 0 } ] }, { - "pattern": ".*" + "name": "winRate", + "displayName": "Win Rate", + "shape": [ + "percent", + { + "decimalPlaces": 1 + } + ] + }, + { + "name": "avgDurationMs", + "displayName": "Avg Build Time", + "shape": "milliseconds" } ] }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "rank", + "title": "#", + "align": "right", + "width": 40, + "renderer": { + "type": "number", + "decimals": 0 + } + }, + { + "field": "url", + "title": "Model", + "width": 240, + "renderer": { + "type": "link", + "labelField": "model" + } + }, + { + "field": "elo", + "title": "Elo", + "align": "right", + "width": 80, + "renderer": { + "type": "text" + } + }, + { + "field": "winRate", + "title": "Win Rate", + "align": "right", + "width": 100, + "renderer": { + "type": "text" + } + }, + { + "field": "avgDurationMs", + "title": "Avg Build Time", + "renderer": { + "type": "bar" + } + } + ] + }, + "style": {} + } + }, "visualisation": { - "type": "data-stream-bar-chart", + "type": "data-stream-table", "config": { - "data-stream-bar-chart": { - "xAxisData": "model_permaslug_uniqueValues", - "yAxisData": [ - "total_tokens_sum" + "data-stream-table": { + "transpose": false, + "columnOrder": [ + "rank", + "model", + "elo", + "winRate", + "avgDurationMs" ], - "xAxisGroup": "none", - "xAxisLabel": "Model", - "yAxisLabel": "Tokens", - "showXAxisLabel": true, - "showYAxisLabel": true, - "showLegend": false, - "legendPosition": "bottom", - "showGrid": true, - "horizontalLayout": "horizontal", - "displayMode": "actual", - "showTotals": false, - "showValue": false, - "grouping": false, - "range": { - "type": "auto" - } + "hiddenColumns": [ + "id", + "url" + ] } } } @@ -397,14 +737,15 @@ }, { "i": "22bac160-9e66-4597-b230-270811258056", - "x": 3, - "y": 7, - "w": 3, - "h": 3, + "x": 4, + "y": 0, + "w": 2, + "h": 4, "config": { "_type": "tile/data-stream", - "title": "OpenRouter Token Share By Author", + "title": "OpenRouter Token Share By Author (This Month)", "description": "", + "timeframe": "thisMonth", "activePluginConfigIds": [ "{{configId}}" ], @@ -462,6 +803,29 @@ } ] }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "pie", + "encoding": { + "label": { + "field": "author_uniqueValues", + "type": "nominal", + "title": "Author" + }, + "value": { + "field": "total_tokens_sum", + "type": "quantitative", + "title": "Tokens" + } + }, + "style": { + "pieInnerRadius": 55, + "pieDonutCenterValue": "total", + "pieLegendStyle": "table" + } + } + }, "visualisation": { "type": "data-stream-donut-chart", "config": { @@ -480,7 +844,7 @@ { "i": "071c926f-0d3c-4192-962f-d191aa03dcbc", "x": 0, - "y": 10, + "y": 11, "w": 6, "h": 4, "config": { @@ -558,28 +922,154 @@ { "i": "a3384d79-77a8-49c0-bb08-62d6744303e1", "x": 0, - "y": 14, + "y": 15, "w": 6, "h": 4, "config": { "_type": "tile/data-stream", - "title": "Top Model By Task", - "description": "", + "title": "Task Leaders", + "description": "Models that are the most used model for at least one task on OpenRouter over the last 7 days, with the tasks they lead.", "timeframe": "none", "activePluginConfigIds": [ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[taskClassifications]}}", - "name": "taskClassifications", - "pluginConfigId": "{{configId}}", - "sort": { - "by": [ - [ - "usageShare", - "desc" - ] + "id": "datastream-sql", + "dataSourceConfig": { + "version": "2.0", + "sql": "-- One row per model that is the most used model for at least one task\nWITH leaders AS (\n SELECT\n topModel AS slug,\n count(*) AS tasksLed,\n string_agg(DISTINCT category, ', ' ORDER BY category) AS categories,\n string_agg(task, ', ' ORDER BY usageShare DESC) AS leads,\n sum(usageShare) AS requestShare\n FROM Tasks\n WHERE topModel IS NOT NULL\n GROUP BY topModel\n),\nby_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY l.tasksLed DESC, l.requestShare DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, l.slug) AS model,\n l.tasksLed,\n l.categories,\n l.leads\nFROM leaders l\nLEFT JOIN by_slug m ON m.slug = l.slug\nORDER BY rank", + "tables": [ + { + "tableName": "Tasks", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[taskClassifications]}}", + "name": "taskClassifications", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } + }, + { + "tableName": "Models", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}" + }, + "timeframe": "none" + } + } ] + }, + "metadata": [ + { + "name": "rank", + "displayName": "Rank", + "shape": "number" + }, + { + "name": "id", + "displayName": "Model ID", + "shape": "string", + "visible": false + }, + { + "name": "url", + "displayName": "Model Link", + "shape": "string", + "visible": false + }, + { + "name": "model", + "displayName": "Model", + "shape": "string", + "role": "label" + }, + { + "name": "tasksLed", + "displayName": "Tasks Led", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "categories", + "displayName": "Categories", + "shape": "string" + }, + { + "name": "leads", + "displayName": "Leads (Busiest First)", + "shape": "string" + } + ] + }, + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "richTable", + "encoding": { + "columns": [ + { + "field": "rank", + "title": "#", + "align": "right", + "width": 40, + "renderer": { + "type": "number", + "decimals": 0 + } + }, + { + "field": "url", + "title": "Model", + "width": 260, + "renderer": { + "type": "link", + "labelField": "model" + } + }, + { + "field": "tasksLed", + "title": "Tasks Led", + "align": "right", + "width": 90, + "renderer": { + "type": "number", + "decimals": 0 + } + }, + { + "field": "categories", + "title": "Categories", + "width": 260, + "renderer": { + "type": "tag-list" + } + }, + { + "field": "leads", + "title": "Leads (Busiest First)", + "renderer": { + "type": "text" + } + } + ] + }, + "style": { + "richTableWrapText": true + } } }, "visualisation": { @@ -588,17 +1078,15 @@ "data-stream-table": { "transpose": false, "columnOrder": [ - "task", - "category", - "usageShare", - "topModel", - "topModelUsageShare", - "asOf" + "rank", + "model", + "tasksLed", + "categories", + "leads" ], "hiddenColumns": [ - "tag", - "tokenShare", - "citation" + "id", + "url" ] } } @@ -608,7 +1096,7 @@ { "i": "37f6bcdf-02a2-4c2a-861d-c9515d1c2092", "x": 0, - "y": 18, + "y": 19, "w": 6, "h": 8, "config": { @@ -697,7 +1185,7 @@ { "i": "56d4d9f8-9cc4-4852-9306-2997d9e87000", "x": 0, - "y": 26, + "y": 27, "w": 6, "h": 8, "config": { @@ -852,7 +1340,7 @@ { "i": "679f0166-77c4-47c1-9ae5-7e4c1721f718", "x": 0, - "y": 34, + "y": 35, "w": 6, "h": 1, "config": { @@ -861,7 +1349,7 @@ "description": "", "visualisation": { "config": { - "content": "Benchmarks: Artificial Analysis ([artificialanalysis.ai](https://artificialanalysis.ai)), Design Arena ([www.designarena.ai](https://www.designarena.ai)), and OpenRouter evals ([openrouter.ai](https://openrouter.ai)), via OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). \nRankings, tasks, and session cost: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). Task data is as of the **As Of** date in the Top Model By Task table; session cost is refreshed weekly. \nLicensed under CC BY 4.0.", + "content": "Benchmarks: Artificial Analysis ([artificialanalysis.ai](https://artificialanalysis.ai)), Design Arena ([www.designarena.ai](https://www.designarena.ai)), and OpenRouter evals ([openrouter.ai](https://openrouter.ai)), via OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). \nRankings, tasks, and session cost: OpenRouter ([openrouter.ai/rankings](https://openrouter.ai/rankings)). Tasks cover the 7 days to yesterday (UTC); session cost is refreshed weekly. \nLicensed under CC BY 4.0.", "fontSize": 12, "align": "left", "autoSize": true diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json index 104f8b09..66bae010 100644 --- a/plugins/OpenRouter/v1/defaultContent/overview.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -536,7 +536,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n sourceId AS id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n m.id,\n COALESCE(m.name, u.slug) AS name,\n u.slug,\n u.spend,\n u.requests,\n u.costPerRequest,\n u.promptTokens,\n u.completionTokens\nFROM Usage u\nLEFT JOIN by_slug m ON m.slug = u.slug\nORDER BY u.spend DESC", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, u.slug) AS name,\n u.slug,\n u.spend,\n u.requests,\n u.costPerRequest,\n u.promptTokens,\n u.completionTokens\nFROM Usage u\nLEFT JOIN by_slug m ON m.slug = u.slug\nORDER BY u.spend DESC", "tables": [ { "tableName": "Usage", @@ -554,13 +554,13 @@ { "tableName": "Models", "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], "dataStream": { - "id": "datastream-properties", - "name": "properties" - }, - "scope": { - "scope": "{{scopes.[Models]}}", - "workspace": "{{workspaceId}}" + "id": "{{dataStreams.[models]}}", + "name": "models", + "pluginConfigId": "{{configId}}" }, "timeframe": "none" } @@ -574,17 +574,18 @@ "shape": "string", "visible": false }, + { + "name": "url", + "displayName": "Model Link", + "shape": "string", + "visible": false + }, { "name": "name", "displayName": "Model", "shape": "string", "role": "label" }, - { - "sourceId": "id", - "sourceType": "Model", - "name": "name" - }, { "name": "slug", "displayName": "Model Slug", @@ -631,10 +632,11 @@ "encoding": { "columns": [ { - "field": "name", + "field": "url", "title": "Model", "renderer": { - "type": "text" + "type": "link", + "labelField": "name" } }, { @@ -699,7 +701,8 @@ ], "hiddenColumns": [ "id", - "slug" + "slug", + "url" ] } } From 416738cfbee9954aa247f3b993bcc2971036e492 Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 16:58:07 +0100 Subject: [PATCH 09/16] Resolve OpenRouter :free models in dataset tiles by permaslug OpenRouter's public datasets key models by permaslug, which keeps the :free suffix, so joining on canonical_slug left free models in rankings and benchmarks without a name or drilldown. Models now carry a permaslug (canonical slug plus :free) that leaderboard lookups and session cost filters join on. Benchmark tiles over selected models stay on canonical_slug, as benchmarks score the weights, not the variant. Co-Authored-By: Claude Opus 5.5 --- plugins/OpenRouter/v1/dataStreams/models.json | 7 +++++++ plugins/OpenRouter/v1/dataStreams/scripts/sessionCost.js | 5 +++-- plugins/OpenRouter/v1/defaultContent/model.dash.json | 5 ++++- .../v1/defaultContent/modelComparison.dash.json | 2 +- .../v1/defaultContent/modelLeaderboards.dash.json | 9 +++++---- plugins/OpenRouter/v1/docs/README.md | 4 ++-- plugins/OpenRouter/v1/indexDefinitions/default.json | 2 +- 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/plugins/OpenRouter/v1/dataStreams/models.json b/plugins/OpenRouter/v1/dataStreams/models.json index 82b5fecb..71234c4c 100644 --- a/plugins/OpenRouter/v1/dataStreams/models.json +++ b/plugins/OpenRouter/v1/dataStreams/models.json @@ -47,6 +47,13 @@ "displayName": "Canonical Slug", "shape": "string" }, + { + "name": "permaslug", + "displayName": "Permaslug", + "shape": "string", + "computed": true, + "valueExpression": "{{ $['canonical_slug'] + ($['id'].endsWith(':free') ? ':free' : '') }}" + }, { "name": "context_length", "displayName": "Context Length", diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/sessionCost.js b/plugins/OpenRouter/v1/dataStreams/scripts/sessionCost.js index 358494e2..7e4e8832 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/sessionCost.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/sessionCost.js @@ -8,9 +8,10 @@ const lengths = { '50-plus-turns': ['50+ turns', 4], }; -// Selected Model objects arrive with each property as a single-element array +// Selected Model objects arrive with each property as a single-element array. +// Match on permaslug, as a :free model's session costs differ from its base's. const unwrap = (v) => (Array.isArray(v) ? v[0] : v); -const slugs = new Set((context.config?.models ?? []).map((o) => unwrap(o.canonical_slug)).filter(Boolean)); +const slugs = new Set((context.config?.models ?? []).map((o) => unwrap(o.permaslug)).filter(Boolean)); const asOf = data?.meta?.as_of; const citation = `Source: OpenRouter (openrouter.ai/rankings), as of ${asOf}. Licensed under CC BY 4.0.`; diff --git a/plugins/OpenRouter/v1/defaultContent/model.dash.json b/plugins/OpenRouter/v1/defaultContent/model.dash.json index 965bdb31..4bd1d9b3 100644 --- a/plugins/OpenRouter/v1/defaultContent/model.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/model.dash.json @@ -98,7 +98,8 @@ "sourceId", "sourceType", "sourceName", - "rawId" + "rawId", + "permaslug" ] } } @@ -266,6 +267,7 @@ "name", "author", "canonical_slug", + "permaslug", "pricing.prompt", "pricing.completion", "pricing.input_cache_read", @@ -1644,6 +1646,7 @@ "name", "author", "canonical_slug", + "permaslug", "pricing.prompt", "pricing.completion", "pricing.input_cache_read", diff --git a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json index 65f44e62..751e280b 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json @@ -826,7 +826,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "WITH by_slug AS (\n SELECT\n canonical_slug,\n sourceId,\n name\n FROM Selected\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n s.name,\n c.app_name AS harness,\n c.median_session_cost_usd AS cost\nFROM SessionCost c\nJOIN by_slug s ON c.model_permaslug = s.canonical_slug\nWHERE c.turn_range = '10-49-turns'\nORDER BY s.name, c.app_name", + "sql": "WITH by_slug AS (\n SELECT\n permaslug,\n sourceId,\n name\n FROM Selected\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(sourceId), sourceId) = 1\n)\nSELECT\n s.name,\n c.app_name AS harness,\n c.median_session_cost_usd AS cost\nFROM SessionCost c\nJOIN by_slug s ON c.model_permaslug = s.permaslug\nWHERE c.turn_range = '10-49-turns'\nORDER BY s.name, c.app_name", "tables": [ { "tableName": "SessionCost", diff --git a/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json b/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json index 0c82a44f..e2f5f09b 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json @@ -25,7 +25,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "-- Month to date vs the same days of last month: last month only counts days up to\n-- the latest day this month has data for, so a partial month compares like for like.\n-- Dates arrive as ISO strings (YYYY-MM-DD...), so month and day are sliced from the text.\n-- Timeframe edges are local time but dates are UTC, so each table can spill a day into the\n-- neighbouring month. Last month is the month most of LastMonth's rows fall in (it is never\n-- empty, unlike ThisMonth on the 1st), and this month is the one after it.\nWITH days AS (\n SELECT 'this' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM ThisMonth\n WHERE model_permaslug <> 'other'\n UNION ALL\n SELECT 'last' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM LastMonth\n WHERE model_permaslug <> 'other'\n),\nprevious AS (\n SELECT mode(substr(day, 1, 7)) AS previous_month\n FROM days\n WHERE source = 'last'\n),\nmonths AS (\n SELECT\n strftime(CAST(previous_month || '-01' AS DATE) + INTERVAL 1 MONTH, '%Y-%m') AS month,\n previous_month\n FROM previous\n),\ncutoff AS (\n SELECT\n m.month,\n m.previous_month,\n (SELECT max(CAST(substr(day, 9, 2) AS INTEGER)) FROM days WHERE source = 'this' AND substr(day, 1, 7) = m.month) AS last_day\n FROM months m\n),\nthis_month AS (\n SELECT d.slug, sum(d.total_tokens) AS tokens\n FROM days d, cutoff\n WHERE d.source = 'this' AND substr(d.day, 1, 7) = cutoff.month\n GROUP BY d.slug\n),\nlast_month AS (\n SELECT\n d.slug,\n sum(d.total_tokens) AS tokens,\n row_number() OVER (ORDER BY sum(d.total_tokens) DESC) AS rank\n FROM days d, cutoff\n WHERE d.source = 'last'\n AND substr(d.day, 1, 7) = cutoff.previous_month\n AND CAST(substr(d.day, 9, 2) AS INTEGER) <= cutoff.last_day\n GROUP BY d.slug\n),\nranked AS (\n SELECT\n slug,\n tokens,\n row_number() OVER (ORDER BY tokens DESC) AS rank\n FROM this_month\n),\nby_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n r.rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, r.slug) AS name,\n r.tokens,\n l.tokens AS lastTokens,\n (r.tokens - l.tokens) / l.tokens AS change,\n COALESCE('#' || CAST(l.rank AS VARCHAR), 'New') AS lastRank,\n l.rank - r.rank AS rankChange\nFROM ranked r\nLEFT JOIN last_month l ON l.slug = r.slug\nLEFT JOIN by_slug m ON m.slug = r.slug\nWHERE r.rank <= 10\nORDER BY r.rank", + "sql": "-- Month to date vs the same days of last month: last month only counts days up to\n-- the latest day this month has data for, so a partial month compares like for like.\n-- Dates arrive as ISO strings (YYYY-MM-DD...), so month and day are sliced from the text.\n-- Timeframe edges are local time but dates are UTC, so each table can spill a day into the\n-- neighbouring month. Last month is the month most of LastMonth's rows fall in (it is never\n-- empty, unlike ThisMonth on the 1st), and this month is the one after it.\nWITH days AS (\n SELECT 'this' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM ThisMonth\n WHERE model_permaslug <> 'other'\n UNION ALL\n SELECT 'last' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM LastMonth\n WHERE model_permaslug <> 'other'\n),\nprevious AS (\n SELECT mode(substr(day, 1, 7)) AS previous_month\n FROM days\n WHERE source = 'last'\n),\nmonths AS (\n SELECT\n strftime(CAST(previous_month || '-01' AS DATE) + INTERVAL 1 MONTH, '%Y-%m') AS month,\n previous_month\n FROM previous\n),\ncutoff AS (\n SELECT\n m.month,\n m.previous_month,\n (SELECT max(CAST(substr(day, 9, 2) AS INTEGER)) FROM days WHERE source = 'this' AND substr(day, 1, 7) = m.month) AS last_day\n FROM months m\n),\nthis_month AS (\n SELECT d.slug, sum(d.total_tokens) AS tokens\n FROM days d, cutoff\n WHERE d.source = 'this' AND substr(d.day, 1, 7) = cutoff.month\n GROUP BY d.slug\n),\nlast_month AS (\n SELECT\n d.slug,\n sum(d.total_tokens) AS tokens,\n row_number() OVER (ORDER BY sum(d.total_tokens) DESC) AS rank\n FROM days d, cutoff\n WHERE d.source = 'last'\n AND substr(d.day, 1, 7) = cutoff.previous_month\n AND CAST(substr(d.day, 9, 2) AS INTEGER) <= cutoff.last_day\n GROUP BY d.slug\n),\nranked AS (\n SELECT\n slug,\n tokens,\n row_number() OVER (ORDER BY tokens DESC) AS rank\n FROM this_month\n),\nby_slug AS (\n SELECT\n permaslug AS slug,\n id,\n name\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\nSELECT\n r.rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, r.slug) AS name,\n r.tokens,\n l.tokens AS lastTokens,\n (r.tokens - l.tokens) / l.tokens AS change,\n COALESCE('#' || CAST(l.rank AS VARCHAR), 'New') AS lastRank,\n l.rank - r.rank AS rankChange\nFROM ranked r\nLEFT JOIN last_month l ON l.slug = r.slug\nLEFT JOIN by_slug m ON m.slug = r.slug\nWHERE r.rank <= 10\nORDER BY r.rank", "tables": [ { "tableName": "ThisMonth", @@ -406,7 +406,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, b.modelName) AS model,\n b.score AS accuracy,\n b.avgCostPerTask\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'gpqa_diamond' AND b.metric = 'accuracy'\nORDER BY b.score DESC\nLIMIT 15", + "sql": "WITH by_slug AS (\n SELECT\n permaslug AS slug,\n id,\n name\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, b.modelName) AS model,\n b.score AS accuracy,\n b.avgCostPerTask\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'gpqa_diamond' AND b.metric = 'accuracy'\nORDER BY b.score DESC\nLIMIT 15", "tables": [ { "tableName": "Benchmarks", @@ -573,7 +573,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n id\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\n-- Design Arena's own names keep reasoning-effort variants apart (e.g. \"(xhigh)\", \"Max\")\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n b.modelName AS model,\n b.score AS elo,\n b.winRate,\n b.avgDurationMs\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'website' AND b.metric = 'elo'\nORDER BY b.score DESC\nLIMIT 15", + "sql": "WITH by_slug AS (\n SELECT\n permaslug AS slug,\n id\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\n-- Design Arena's own names keep reasoning-effort variants apart (e.g. \"(xhigh)\", \"Max\")\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n b.modelName AS model,\n b.score AS elo,\n b.winRate,\n b.avgDurationMs\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'website' AND b.metric = 'elo'\nORDER BY b.score DESC\nLIMIT 15", "tables": [ { "tableName": "Benchmarks", @@ -937,7 +937,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "-- One row per model that is the most used model for at least one task\nWITH leaders AS (\n SELECT\n topModel AS slug,\n count(*) AS tasksLed,\n string_agg(DISTINCT category, ', ' ORDER BY category) AS categories,\n string_agg(task, ', ' ORDER BY usageShare DESC) AS leads,\n sum(usageShare) AS requestShare\n FROM Tasks\n WHERE topModel IS NOT NULL\n GROUP BY topModel\n),\nby_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY l.tasksLed DESC, l.requestShare DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, l.slug) AS model,\n l.tasksLed,\n l.categories,\n l.leads\nFROM leaders l\nLEFT JOIN by_slug m ON m.slug = l.slug\nORDER BY rank", + "sql": "-- One row per model that is the most used model for at least one task\nWITH leaders AS (\n SELECT\n topModel AS slug,\n count(*) AS tasksLed,\n string_agg(DISTINCT category, ', ' ORDER BY category) AS categories,\n string_agg(task, ', ' ORDER BY usageShare DESC) AS leads,\n sum(usageShare) AS requestShare\n FROM Tasks\n WHERE topModel IS NOT NULL\n GROUP BY topModel\n),\nby_slug AS (\n SELECT\n permaslug AS slug,\n id,\n name\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY l.tasksLed DESC, l.requestShare DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, l.slug) AS model,\n l.tasksLed,\n l.categories,\n l.leads\nFROM leaders l\nLEFT JOIN by_slug m ON m.slug = l.slug\nORDER BY rank", "tables": [ { "tableName": "Tasks", @@ -1316,6 +1316,7 @@ "hiddenColumns": [ "id", "canonical_slug", + "permaslug", "cacheReadPrice", "top_provider.max_completion_tokens", "inputModalities", diff --git a/plugins/OpenRouter/v1/docs/README.md b/plugins/OpenRouter/v1/docs/README.md index 32418305..b9c878f8 100644 --- a/plugins/OpenRouter/v1/docs/README.md +++ b/plugins/OpenRouter/v1/docs/README.md @@ -39,11 +39,11 @@ The out-of-the-box dashboards include an account **Overview**; **Model Leaderboa ## Data streams - **Usage** — spend, cost per 1M tokens, requests, tokens, cache hit rate, latency, time to first token, or throughput over time, optionally grouped by model, provider, workspace, API key, app, user, origin, country, finish reason, or streamed, and optionally filtered to selected models, providers, workspaces, or API keys. Any other OpenRouter analytics metric or dimension can be typed in. Account-wide. Management key only. -- **Model Usage** — spend, requests, cost per request, and prompt and completion tokens for every model over the timeframe, keyed by canonical slug. OpenRouter reports `:batch` variants under the same slug as their base model, so their usage is combined. The overview dashboard joins this to the imported models for names and drilldown. Account-wide. Management key only. +- **Model Usage** — spend, requests, cost per request, and prompt and completion tokens for every model over the timeframe, keyed by canonical slug. OpenRouter reports `:free` and `:batch` variants under the same slug as their base model, so their usage is combined. The overview dashboard joins this to the imported models for names and drilldown. Account-wide. Management key only. - **API Keys** — API keys in a workspace with spend and limits. Per workspace. Management key only. - **Workspaces** — workspaces in the account with their default models, provider sort, disabled server tools, whether BYOK spend counts toward budgets, and logging and broadcast settings, optionally filtered to selected workspaces. Account-wide. Management key only. - **Credits** — credits purchased, used, and remaining. Account-wide. -- **Models** — every model on OpenRouter with pricing ($ per 1M tokens), context length, modalities, Artificial Analysis index scores, and affordability scores (0 to 100, higher is cheaper), optionally filtered to selected models. Prices score on a log scale from 100 at $0.01 per 1M tokens to 0 at the **Affordability ceiling** parameter ($1,000 by default, minimum $1). +- **Models** — every model on OpenRouter with pricing ($ per 1M tokens), context length, modalities, Artificial Analysis index scores, and affordability scores (0 to 100, higher is cheaper), optionally filtered to selected models. Prices score on a log scale from 100 at $0.01 per 1M tokens to 0 at the **Affordability ceiling** parameter ($1,000 by default, minimum $1). Includes a permaslug (canonical slug, plus `:free` for free variants) that matches how OpenRouter's public datasets name models. - **Providers** — every provider on OpenRouter, with headquarters and datacenter countries (by name), and status, privacy, and terms links. - **Model Providers** — providers serving a model, with status, price (including cache reads), uptime over the last 5 minutes, 30 minutes, and day, and latency (p50 to p99) and throughput over the last 30 minutes. Per model. - **Benchmarks** — model benchmark scores from Artificial Analysis, Design Arena, and OpenRouter evals, one row per model and benchmark. diff --git a/plugins/OpenRouter/v1/indexDefinitions/default.json b/plugins/OpenRouter/v1/indexDefinitions/default.json index 00b5b253..8a1fbc79 100644 --- a/plugins/OpenRouter/v1/indexDefinitions/default.json +++ b/plugins/OpenRouter/v1/indexDefinitions/default.json @@ -12,7 +12,7 @@ "type": { "value": "Model" }, - "properties": ["author", "canonical_slug", "context_length"] + "properties": ["author", "canonical_slug", "permaslug", "context_length"] } }, { From ecfdc269c12c1a78898b72676bc0135ca4d5a99d Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 17:00:18 +0100 Subject: [PATCH 10/16] Link uncatalogued OpenRouter models to their OpenRouter page Lookup tiles only built a drilldown url when the slug matched a Model, so models missing from the catalog (e.g. stealth models) rendered a broken relative link. They now fall back to openrouter.ai/, which serves a page for dated and :free slugs alike. Co-Authored-By: Claude Opus 5.5 --- .../v1/defaultContent/modelLeaderboards.dash.json | 8 ++++---- plugins/OpenRouter/v1/defaultContent/overview.dash.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json b/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json index e2f5f09b..bdc16f6b 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelLeaderboards.dash.json @@ -25,7 +25,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "-- Month to date vs the same days of last month: last month only counts days up to\n-- the latest day this month has data for, so a partial month compares like for like.\n-- Dates arrive as ISO strings (YYYY-MM-DD...), so month and day are sliced from the text.\n-- Timeframe edges are local time but dates are UTC, so each table can spill a day into the\n-- neighbouring month. Last month is the month most of LastMonth's rows fall in (it is never\n-- empty, unlike ThisMonth on the 1st), and this month is the one after it.\nWITH days AS (\n SELECT 'this' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM ThisMonth\n WHERE model_permaslug <> 'other'\n UNION ALL\n SELECT 'last' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM LastMonth\n WHERE model_permaslug <> 'other'\n),\nprevious AS (\n SELECT mode(substr(day, 1, 7)) AS previous_month\n FROM days\n WHERE source = 'last'\n),\nmonths AS (\n SELECT\n strftime(CAST(previous_month || '-01' AS DATE) + INTERVAL 1 MONTH, '%Y-%m') AS month,\n previous_month\n FROM previous\n),\ncutoff AS (\n SELECT\n m.month,\n m.previous_month,\n (SELECT max(CAST(substr(day, 9, 2) AS INTEGER)) FROM days WHERE source = 'this' AND substr(day, 1, 7) = m.month) AS last_day\n FROM months m\n),\nthis_month AS (\n SELECT d.slug, sum(d.total_tokens) AS tokens\n FROM days d, cutoff\n WHERE d.source = 'this' AND substr(d.day, 1, 7) = cutoff.month\n GROUP BY d.slug\n),\nlast_month AS (\n SELECT\n d.slug,\n sum(d.total_tokens) AS tokens,\n row_number() OVER (ORDER BY sum(d.total_tokens) DESC) AS rank\n FROM days d, cutoff\n WHERE d.source = 'last'\n AND substr(d.day, 1, 7) = cutoff.previous_month\n AND CAST(substr(d.day, 9, 2) AS INTEGER) <= cutoff.last_day\n GROUP BY d.slug\n),\nranked AS (\n SELECT\n slug,\n tokens,\n row_number() OVER (ORDER BY tokens DESC) AS rank\n FROM this_month\n),\nby_slug AS (\n SELECT\n permaslug AS slug,\n id,\n name\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\nSELECT\n r.rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, r.slug) AS name,\n r.tokens,\n l.tokens AS lastTokens,\n (r.tokens - l.tokens) / l.tokens AS change,\n COALESCE('#' || CAST(l.rank AS VARCHAR), 'New') AS lastRank,\n l.rank - r.rank AS rankChange\nFROM ranked r\nLEFT JOIN last_month l ON l.slug = r.slug\nLEFT JOIN by_slug m ON m.slug = r.slug\nWHERE r.rank <= 10\nORDER BY r.rank", + "sql": "-- Month to date vs the same days of last month: last month only counts days up to\n-- the latest day this month has data for, so a partial month compares like for like.\n-- Dates arrive as ISO strings (YYYY-MM-DD...), so month and day are sliced from the text.\n-- Timeframe edges are local time but dates are UTC, so each table can spill a day into the\n-- neighbouring month. Last month is the month most of LastMonth's rows fall in (it is never\n-- empty, unlike ThisMonth on the 1st), and this month is the one after it.\nWITH days AS (\n SELECT 'this' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM ThisMonth\n WHERE model_permaslug <> 'other'\n UNION ALL\n SELECT 'last' AS source, CAST(date AS VARCHAR) AS day, model_permaslug AS slug, total_tokens\n FROM LastMonth\n WHERE model_permaslug <> 'other'\n),\nprevious AS (\n SELECT mode(substr(day, 1, 7)) AS previous_month\n FROM days\n WHERE source = 'last'\n),\nmonths AS (\n SELECT\n strftime(CAST(previous_month || '-01' AS DATE) + INTERVAL 1 MONTH, '%Y-%m') AS month,\n previous_month\n FROM previous\n),\ncutoff AS (\n SELECT\n m.month,\n m.previous_month,\n (SELECT max(CAST(substr(day, 9, 2) AS INTEGER)) FROM days WHERE source = 'this' AND substr(day, 1, 7) = m.month) AS last_day\n FROM months m\n),\nthis_month AS (\n SELECT d.slug, sum(d.total_tokens) AS tokens\n FROM days d, cutoff\n WHERE d.source = 'this' AND substr(d.day, 1, 7) = cutoff.month\n GROUP BY d.slug\n),\nlast_month AS (\n SELECT\n d.slug,\n sum(d.total_tokens) AS tokens,\n row_number() OVER (ORDER BY sum(d.total_tokens) DESC) AS rank\n FROM days d, cutoff\n WHERE d.source = 'last'\n AND substr(d.day, 1, 7) = cutoff.previous_month\n AND CAST(substr(d.day, 9, 2) AS INTEGER) <= cutoff.last_day\n GROUP BY d.slug\n),\nranked AS (\n SELECT\n slug,\n tokens,\n row_number() OVER (ORDER BY tokens DESC) AS rank\n FROM this_month\n),\nby_slug AS (\n SELECT\n permaslug AS slug,\n id,\n name\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\nSELECT\n r.rank,\n m.id,\n -- Models missing from the models stream (e.g. stealth models) link to their OpenRouter page\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' ELSE 'https://openrouter.ai/' || r.slug END AS url,\n COALESCE(m.name, r.slug) AS name,\n r.tokens,\n l.tokens AS lastTokens,\n (r.tokens - l.tokens) / l.tokens AS change,\n COALESCE('#' || CAST(l.rank AS VARCHAR), 'New') AS lastRank,\n l.rank - r.rank AS rankChange\nFROM ranked r\nLEFT JOIN last_month l ON l.slug = r.slug\nLEFT JOIN by_slug m ON m.slug = r.slug\nWHERE r.rank <= 10\nORDER BY r.rank", "tables": [ { "tableName": "ThisMonth", @@ -406,7 +406,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "WITH by_slug AS (\n SELECT\n permaslug AS slug,\n id,\n name\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, b.modelName) AS model,\n b.score AS accuracy,\n b.avgCostPerTask\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'gpqa_diamond' AND b.metric = 'accuracy'\nORDER BY b.score DESC\nLIMIT 15", + "sql": "WITH by_slug AS (\n SELECT\n permaslug AS slug,\n id,\n name\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n -- Models missing from the models stream (e.g. stealth models) link to their OpenRouter page\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' ELSE 'https://openrouter.ai/' || b.model END AS url,\n COALESCE(m.name, b.modelName) AS model,\n b.score AS accuracy,\n b.avgCostPerTask\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'gpqa_diamond' AND b.metric = 'accuracy'\nORDER BY b.score DESC\nLIMIT 15", "tables": [ { "tableName": "Benchmarks", @@ -573,7 +573,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "WITH by_slug AS (\n SELECT\n permaslug AS slug,\n id\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\n-- Design Arena's own names keep reasoning-effort variants apart (e.g. \"(xhigh)\", \"Max\")\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n b.modelName AS model,\n b.score AS elo,\n b.winRate,\n b.avgDurationMs\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'website' AND b.metric = 'elo'\nORDER BY b.score DESC\nLIMIT 15", + "sql": "WITH by_slug AS (\n SELECT\n permaslug AS slug,\n id\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\n-- Design Arena's own names keep reasoning-effort variants apart (e.g. \"(xhigh)\", \"Max\")\nSELECT\n row_number() OVER (ORDER BY b.score DESC) AS rank,\n m.id,\n -- Models missing from the models stream (e.g. stealth models) link to their OpenRouter page\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' ELSE 'https://openrouter.ai/' || b.model END AS url,\n b.modelName AS model,\n b.score AS elo,\n b.winRate,\n b.avgDurationMs\nFROM Benchmarks b\nLEFT JOIN by_slug m ON m.slug = b.model\nWHERE b.benchmark = 'website' AND b.metric = 'elo'\nORDER BY b.score DESC\nLIMIT 15", "tables": [ { "tableName": "Benchmarks", @@ -937,7 +937,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "-- One row per model that is the most used model for at least one task\nWITH leaders AS (\n SELECT\n topModel AS slug,\n count(*) AS tasksLed,\n string_agg(DISTINCT category, ', ' ORDER BY category) AS categories,\n string_agg(task, ', ' ORDER BY usageShare DESC) AS leads,\n sum(usageShare) AS requestShare\n FROM Tasks\n WHERE topModel IS NOT NULL\n GROUP BY topModel\n),\nby_slug AS (\n SELECT\n permaslug AS slug,\n id,\n name\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY l.tasksLed DESC, l.requestShare DESC) AS rank,\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, l.slug) AS model,\n l.tasksLed,\n l.categories,\n l.leads\nFROM leaders l\nLEFT JOIN by_slug m ON m.slug = l.slug\nORDER BY rank", + "sql": "-- One row per model that is the most used model for at least one task\nWITH leaders AS (\n SELECT\n topModel AS slug,\n count(*) AS tasksLed,\n string_agg(DISTINCT category, ', ' ORDER BY category) AS categories,\n string_agg(task, ', ' ORDER BY usageShare DESC) AS leads,\n sum(usageShare) AS requestShare\n FROM Tasks\n WHERE topModel IS NOT NULL\n GROUP BY topModel\n),\nby_slug AS (\n SELECT\n permaslug AS slug,\n id,\n name\n FROM Models\n -- Datasets key models by permaslug; a :batch model shares its base's, so keep one\n QUALIFY row_number() OVER (PARTITION BY permaslug ORDER BY length(id), id) = 1\n)\nSELECT\n row_number() OVER (ORDER BY l.tasksLed DESC, l.requestShare DESC) AS rank,\n m.id,\n -- Models missing from the models stream (e.g. stealth models) link to their OpenRouter page\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' ELSE 'https://openrouter.ai/' || l.slug END AS url,\n COALESCE(m.name, l.slug) AS model,\n l.tasksLed,\n l.categories,\n l.leads\nFROM leaders l\nLEFT JOIN by_slug m ON m.slug = l.slug\nORDER BY rank", "tables": [ { "tableName": "Tasks", diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json index 66bae010..4f4c4641 100644 --- a/plugins/OpenRouter/v1/defaultContent/overview.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -536,7 +536,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n m.id,\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' END AS url,\n COALESCE(m.name, u.slug) AS name,\n u.slug,\n u.spend,\n u.requests,\n u.costPerRequest,\n u.promptTokens,\n u.completionTokens\nFROM Usage u\nLEFT JOIN by_slug m ON m.slug = u.slug\nORDER BY u.spend DESC", + "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n m.id,\n -- Models missing from the models stream (e.g. stealth models) link to their OpenRouter page\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' ELSE 'https://openrouter.ai/' || u.slug END AS url,\n COALESCE(m.name, u.slug) AS name,\n u.slug,\n u.spend,\n u.requests,\n u.costPerRequest,\n u.promptTokens,\n u.completionTokens\nFROM Usage u\nLEFT JOIN by_slug m ON m.slug = u.slug\nORDER BY u.spend DESC", "tables": [ { "tableName": "Usage", From 0789adb0976f9d63e6d365554aa8a24d8854034f Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 17:09:35 +0100 Subject: [PATCH 11/16] Return OpenRouter usage as one column per metric The usage stream returned a single chosen metric as `value`, so every tile picked a metric parameter and the Model Usage table needed its own modelUsage stream to get several metrics per model. Usage now returns every additive metric as its own column, and tiles pick the column they plot; the Model Usage table sums per model in SQL instead. Latency, time to first token, and throughput move to a performance stream: percentiles can't be summed, and OpenRouter only serves them for 31 days, which would otherwise cap every usage timeframe. Co-Authored-By: Claude Opus 5.5 --- .../OpenRouter/v1/dataStreams/modelUsage.json | 82 --------- .../v1/dataStreams/performance.json | 161 ++++++++++++++++ .../v1/dataStreams/scripts/modelUsage.js | 27 --- .../v1/dataStreams/scripts/usage.js | 40 ++-- plugins/OpenRouter/v1/dataStreams/usage.json | 174 ++++++++---------- .../v1/defaultContent/apiKey.dash.json | 25 ++- .../v1/defaultContent/model.dash.json | 50 +++-- .../defaultContent/modelComparison.dash.json | 14 +- .../v1/defaultContent/overview.dash.json | 106 +++++------ .../v1/defaultContent/provider.dash.json | 47 +++-- .../v1/defaultContent/workspace.dash.json | 36 ++-- plugins/OpenRouter/v1/docs/README.md | 6 +- 12 files changed, 384 insertions(+), 384 deletions(-) delete mode 100644 plugins/OpenRouter/v1/dataStreams/modelUsage.json create mode 100644 plugins/OpenRouter/v1/dataStreams/performance.json delete mode 100644 plugins/OpenRouter/v1/dataStreams/scripts/modelUsage.js diff --git a/plugins/OpenRouter/v1/dataStreams/modelUsage.json b/plugins/OpenRouter/v1/dataStreams/modelUsage.json deleted file mode 100644 index 3ecac5a7..00000000 --- a/plugins/OpenRouter/v1/dataStreams/modelUsage.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "name": "modelUsage", - "displayName": "Model Usage", - "description": "Spend, requests, cost per request, and tokens for each model over the timeframe, by canonical slug (requires a management key)", - "tags": [ - "Usage", - "Cost" - ], - "baseDataSourceName": "httpRequestUnscoped", - "config": { - "httpMethod": "post", - "endpointPath": "analytics/query", - "paging": { - "mode": "none" - }, - "getArgs": [], - "headers": [], - "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; return JSON.stringify({ metrics: ['total_usage', 'request_count', 'tokens_prompt', 'tokens_completion'], dimensions: ['model'], granularity: 'month', time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", - "postRequestScript": "modelUsage.js" - }, - "matches": "none", - "metadata": [ - { - "name": "slug", - "displayName": "Model", - "shape": "string", - "role": "label" - }, - { - "name": "spend", - "displayName": "Spend", - "shape": "usd", - "role": "value" - }, - { - "name": "requests", - "displayName": "Requests", - "shape": "number" - }, - { - "name": "costPerRequest", - "displayName": "Cost Per Request", - "shape": [ - "usd", - { - "decimalPlaces": 6 - } - ] - }, - { - "name": "promptTokens", - "displayName": "Prompt Tokens", - "shape": "number" - }, - { - "name": "completionTokens", - "displayName": "Completion Tokens", - "shape": "number" - } - ], - "timeframes": [ - "last12hours", - "last24hours", - "last7days", - "last30days", - "thisMonth", - "lastMonth", - "thisQuarter", - "lastQuarter", - "thisYear" - ], - "defaultShaping": { - "sort": { - "by": [ - [ - "spend", - "desc" - ] - ] - } - } -} diff --git a/plugins/OpenRouter/v1/dataStreams/performance.json b/plugins/OpenRouter/v1/dataStreams/performance.json new file mode 100644 index 00000000..baa9498e --- /dev/null +++ b/plugins/OpenRouter/v1/dataStreams/performance.json @@ -0,0 +1,161 @@ +{ + "name": "performance", + "displayName": "Performance", + "description": "Latency, time to first token, and throughput percentiles over time, optionally grouped by model, provider, workspace, API key, or app (last 30 days at most; requires a management key)", + "tags": [ + "Usage", + "Performance" + ], + "baseDataSourceName": "httpRequestUnscoped", + "config": { + "httpMethod": "post", + "endpointPath": "analytics/query", + "paging": { + "mode": "none" + }, + "getArgs": [], + "headers": [], + "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; const days = (new Date(timeframe.end) - new Date(timeframe.start)) / 864e5; const granularity = days <= 2 ? 'hour' : days <= 31 ? 'day' : days <= 92 ? 'week' : 'month'; const selected = typeof filterObjects !== 'undefined' && Array.isArray(filterObjects) ? filterObjects : []; const one = (v) => (Array.isArray(v) ? v[0] : v); const idsOf = (type, ...props) => [...new Set(selected.filter((o) => one(o.sourceType) === type).flatMap((o) => props.map((p) => one(o[p]))).filter(Boolean))]; const rules = [ ['model', idsOf('Model', 'rawId', 'canonical_slug')], ['provider', idsOf('Provider', 'name')], ['workspace', idsOf('Workspace', 'rawId')], ['api_key_id', idsOf('API Key', 'rawId')] ]; const filters = rules.filter(([, values]) => values.length).map(([field, value]) => ({ field, operator: 'in', value })); return JSON.stringify({ metrics: ['p50_latency', 'p90_latency', 'p50_total_time_to_first_token', 'p90_total_time_to_first_token', 'p50_throughput'], dimensions: typeof groupBy !== 'undefined' && groupBy ? [groupBy] : [], filters, granularity, time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", + "postRequestScript": "usage.js" + }, + "matches": "none", + "ui": [ + { + "type": "autocomplete", + "name": "groupBy", + "label": "Group by", + "help": "Select a dimension, or enter any dimension from [OpenRouter's analytics dimensions](https://openrouter.ai/docs/api/api-reference/analytics/get-available-analytics-metrics-and-dimensions).", + "allowCustomValues": true, + "isMulti": false, + "isClearable": true, + "data": { + "source": "fixed", + "values": [ + { + "value": "model", + "label": "Model" + }, + { + "value": "workspace", + "label": "Workspace" + }, + { + "value": "api_key_id", + "label": "API key" + }, + { + "value": "app", + "label": "App" + }, + { + "value": "user", + "label": "User" + }, + { + "value": "provider", + "label": "Provider" + }, + { + "value": "origin", + "label": "Origin" + }, + { + "value": "country", + "label": "Country" + }, + { + "value": "finish_reason", + "label": "Finish reason" + }, + { + "value": "streamed", + "label": "Streamed" + } + ] + } + }, + { + "type": "objects", + "name": "filterObjects", + "label": "Filter to objects (optional)", + "matches": { + "sourceType": { + "type": "oneOf", + "values": [ + "Model", + "Provider", + "Workspace", + "API Key" + ] + } + } + } + ], + "metadata": [ + { + "name": "date", + "displayName": "Date", + "shape": [ + "date", + { + "timeZone": "Etc/UTC" + } + ], + "role": "timestamp" + }, + { + "name": "group", + "displayName": "Group", + "shape": "string", + "role": "label" + }, + { + "name": "p50_latency", + "displayName": "Latency p50", + "shape": "milliseconds", + "role": "value" + }, + { + "name": "p90_latency", + "displayName": "Latency p90", + "shape": "milliseconds" + }, + { + "name": "p50_total_time_to_first_token", + "displayName": "Time To First Token p50", + "shape": "milliseconds" + }, + { + "name": "p90_total_time_to_first_token", + "displayName": "Time To First Token p90", + "shape": "milliseconds" + }, + { + "name": "p50_throughput", + "displayName": "Throughput p50 (Tokens/s)", + "shape": [ + "number", + { + "decimalPlaces": 1 + } + ] + } + ], + "timeframes": [ + "last12hours", + "last24hours", + "last7days", + "last30days", + "thisMonth" + ], + "defaultShaping": { + "sort": { + "by": [ + [ + "date", + "asc" + ] + ] + } + } +} diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/modelUsage.js b/plugins/OpenRouter/v1/dataStreams/scripts/modelUsage.js deleted file mode 100644 index b90ec13e..00000000 --- a/plugins/OpenRouter/v1/dataStreams/scripts/modelUsage.js +++ /dev/null @@ -1,27 +0,0 @@ -// Sums analytics rows per model. Analytics reports models by canonical slug -// (e.g. z-ai/glm-5.3-flash-20260826), not by Model object id; dashboards join -// on the Model objects' canonical_slug property to resolve names and drilldowns. -const body = data?.data ?? {}; - -if (body.metadata?.truncated) { - api.report.warning('Results were truncated by OpenRouter; narrow the timeframe'); -} - -const totals = new Map(); -for (const row of body.data ?? []) { - const slug = row.model; - if (!slug) continue; - - const total = totals.get(slug) ?? { spend: 0, requests: 0, promptTokens: 0, completionTokens: 0 }; - total.spend += Number(row.total_usage ?? 0); - total.requests += Number(row.request_count ?? 0); - total.promptTokens += Number(row.tokens_prompt ?? 0); - total.completionTokens += Number(row.tokens_completion ?? 0); - totals.set(slug, total); -} - -result = [...totals].map(([slug, total]) => ({ - slug, - ...total, - costPerRequest: total.requests ? total.spend / total.requests : null, -})); diff --git a/plugins/OpenRouter/v1/dataStreams/scripts/usage.js b/plugins/OpenRouter/v1/dataStreams/scripts/usage.js index 7e976425..642bdcea 100644 --- a/plugins/OpenRouter/v1/dataStreams/scripts/usage.js +++ b/plugins/OpenRouter/v1/dataStreams/scripts/usage.js @@ -1,9 +1,10 @@ -// Normalises analytics rows: the time key is suffixed with the granularity -// (date__day, or created_at__day for raw-view fields like latency and provider) -// and large counts arrive as strings. cost_per_million_tokens isn't an OpenRouter -// metric; it's derived from total_usage and tokens_total, requested together. +// Normalises analytics rows for the usage and performance streams: one column +// per requested metric, named as OpenRouter names it. The time key is suffixed +// with the granularity (date__day, or created_at__day for raw-view fields like +// latency and provider) and large counts arrive as strings. +// cost_per_million_tokens isn't an OpenRouter metric; it's derived per row when +// total_usage and tokens_total are both requested. const body = data?.data ?? {}; -const metric = context.config.metric || 'total_usage'; const groupBy = context.config.groupBy; if (body.metadata?.truncated) { @@ -12,24 +13,23 @@ if (body.metadata?.truncated) { ); } -const costPerMillionTokens = (row) => { - const tokens = Number(row.tokens_total ?? 0); - return tokens && row.total_usage != null ? (Number(row.total_usage) / tokens) * 1e6 : null; -}; - -const dateKey = (row) => Object.keys(row).find((k) => /__(minute|hour|day|week|month)$/.test(k)); +const isDateKey = (key) => /__(minute|hour|day|week|month)$/.test(key); result = (body.data ?? []) .map((row) => { - const date = row[dateKey(row)]; - if (!date) return null; + const out = { date: null, group: groupBy ? row[groupBy] ?? null : null }; + + // Numeric strings become numbers; anything else passes through untouched + for (const [key, value] of Object.entries(row)) { + if (isDateKey(key)) out.date = value; + else if (key !== groupBy) out[key] = value != null && Number.isFinite(Number(value)) ? Number(value) : value; + } - const value = metric === 'cost_per_million_tokens' ? costPerMillionTokens(row) : row[metric]; + if ('total_usage' in out && 'tokens_total' in out) { + out.cost_per_million_tokens = + out.tokens_total && out.total_usage != null ? (out.total_usage / out.tokens_total) * 1e6 : null; + } - return { - date, - group: groupBy ? row[groupBy] : null, - value: value == null ? null : Number(value), - }; + return out; }) - .filter(Boolean); + .filter((row) => row.date); diff --git a/plugins/OpenRouter/v1/dataStreams/usage.json b/plugins/OpenRouter/v1/dataStreams/usage.json index 1807112c..2e6e1cf1 100644 --- a/plugins/OpenRouter/v1/dataStreams/usage.json +++ b/plugins/OpenRouter/v1/dataStreams/usage.json @@ -1,7 +1,7 @@ { "name": "usage", "displayName": "Usage", - "description": "Spend, requests, tokens, latency, and throughput over time, optionally grouped by model, provider, workspace, API key, or app (requires a management key)", + "description": "Spend, requests, and tokens over time, optionally grouped by model, provider, workspace, API key, or app (requires a management key)", "tags": [ "Usage", "Cost" @@ -15,103 +15,11 @@ }, "getArgs": [], "headers": [], - "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; const days = (new Date(timeframe.end) - new Date(timeframe.start)) / 864e5; const granularity = days <= 2 ? 'hour' : days <= 31 ? 'day' : days <= 92 ? 'week' : 'month'; const selected = typeof filterObjects !== 'undefined' && Array.isArray(filterObjects) ? filterObjects : []; const one = (v) => (Array.isArray(v) ? v[0] : v); const idsOf = (type, ...props) => [...new Set(selected.filter((o) => one(o.sourceType) === type).flatMap((o) => props.map((p) => one(o[p]))).filter(Boolean))]; const rules = [ ['model', idsOf('Model', 'rawId', 'canonical_slug')], ['provider', idsOf('Provider', 'name')], ['workspace', idsOf('Workspace', 'rawId')], ['api_key_id', idsOf('API Key', 'rawId')] ]; const filters = rules.filter(([, values]) => values.length).map(([field, value]) => ({ field, operator: 'in', value })); const selectedMetric = typeof metric !== 'undefined' && metric ? metric : 'total_usage'; return JSON.stringify({ metrics: selectedMetric === 'cost_per_million_tokens' ? ['total_usage', 'tokens_total'] : [selectedMetric], dimensions: typeof groupBy !== 'undefined' && groupBy ? [groupBy] : [], filters, granularity, time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", + "postBody": "{{ (() => { const iso = (d) => new Date(d).toISOString().slice(0, 19) + 'Z'; const days = (new Date(timeframe.end) - new Date(timeframe.start)) / 864e5; const granularity = days <= 2 ? 'hour' : days <= 31 ? 'day' : days <= 92 ? 'week' : 'month'; const selected = typeof filterObjects !== 'undefined' && Array.isArray(filterObjects) ? filterObjects : []; const one = (v) => (Array.isArray(v) ? v[0] : v); const idsOf = (type, ...props) => [...new Set(selected.filter((o) => one(o.sourceType) === type).flatMap((o) => props.map((p) => one(o[p]))).filter(Boolean))]; const rules = [ ['model', idsOf('Model', 'rawId', 'canonical_slug')], ['provider', idsOf('Provider', 'name')], ['workspace', idsOf('Workspace', 'rawId')], ['api_key_id', idsOf('API Key', 'rawId')] ]; const filters = rules.filter(([, values]) => values.length).map(([field, value]) => ({ field, operator: 'in', value })); return JSON.stringify({ metrics: ['total_usage', 'byok_usage', 'request_count', 'tokens_total', 'tokens_prompt', 'tokens_completion', 'reasoning_tokens', 'cached_tokens'], dimensions: typeof groupBy !== 'undefined' && groupBy ? [groupBy] : [], filters, granularity, time_range: { start: iso(timeframe.start), end: iso(timeframe.end) } }); })() }}", "postRequestScript": "usage.js" }, "matches": "none", "ui": [ - { - "type": "autocomplete", - "name": "metric", - "label": "Metric", - "defaultValue": "total_usage", - "help": "Select a metric, or enter any metric from [OpenRouter's analytics metrics](https://openrouter.ai/docs/api/api-reference/analytics/get-available-analytics-metrics-and-dimensions). Performance metrics are limited to 31 days.", - "validation": { - "required": true - }, - "allowCustomValues": true, - "isMulti": false, - "data": { - "source": "fixed", - "values": [ - { - "label": "Cost", - "options": [ - { - "value": "total_usage", - "label": "Spend (USD)" - }, - { - "value": "byok_usage", - "label": "BYOK spend (USD)" - }, - { - "value": "cost_per_million_tokens", - "label": "Cost per 1M tokens (USD)" - } - ] - }, - { - "label": "Requests and tokens", - "options": [ - { - "value": "request_count", - "label": "Requests" - }, - { - "value": "tokens_total", - "label": "Total tokens" - }, - { - "value": "tokens_prompt", - "label": "Prompt tokens" - }, - { - "value": "tokens_completion", - "label": "Completion tokens" - }, - { - "value": "reasoning_tokens", - "label": "Reasoning tokens" - }, - { - "value": "cached_tokens", - "label": "Cached tokens" - }, - { - "value": "cache_hit_rate", - "label": "Cache hit rate" - } - ] - }, - { - "label": "Performance (31 days max)", - "options": [ - { - "value": "p50_latency", - "label": "Latency p50 (ms)" - }, - { - "value": "p90_latency", - "label": "Latency p90 (ms)" - }, - { - "value": "p50_total_time_to_first_token", - "label": "Time to first token p50 (ms)" - }, - { - "value": "p90_total_time_to_first_token", - "label": "Time to first token p90 (ms)" - }, - { - "value": "p50_throughput", - "label": "Throughput p50 (tokens/s)" - } - ] - } - ] - } - }, { "type": "autocomplete", "name": "groupBy", @@ -212,15 +120,85 @@ "role": "label" }, { - "name": "value", - "displayName": "Value", + "name": "total_usage", + "displayName": "Spend", + "shape": "usd", + "role": "value" + }, + { + "name": "byok_usage", + "displayName": "BYOK Spend", + "shape": "usd" + }, + { + "name": "request_count", + "displayName": "Requests", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "tokens_total", + "displayName": "Total Tokens", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "tokens_prompt", + "displayName": "Prompt Tokens", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "tokens_completion", + "displayName": "Completion Tokens", "shape": [ "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "reasoning_tokens", + "displayName": "Reasoning Tokens", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "cached_tokens", + "displayName": "Cached Tokens", + "shape": [ + "number", + { + "decimalPlaces": 0 + } + ] + }, + { + "name": "cost_per_million_tokens", + "displayName": "Cost Per 1M Tokens", + "shape": [ + "usd", { "decimalPlaces": 4 } - ], - "role": "value" + ] } ], "timeframes": [ diff --git a/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json index de1ab824..d19dac53 100644 --- a/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json @@ -166,7 +166,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "filterObjects": { "scope": "{{scopes.[API Keys]}}", "workspace": "{{workspaceId}}", @@ -183,7 +182,7 @@ }, "metadata": [ { - "name": "value", + "name": "total_usage", "shape": [ "currency", { @@ -211,7 +210,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "total_usage", "type": "quantitative", "title": "Spend ($)" } @@ -228,7 +227,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "total_usage" ], "seriesColumn": "none", "showLegend": false, @@ -262,7 +261,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "request_count", "filterObjects": { "scope": "{{scopes.[API Keys]}}", "workspace": "{{workspaceId}}", @@ -279,7 +277,7 @@ }, "metadata": [ { - "name": "value", + "name": "request_count", "shape": [ "number", { @@ -307,7 +305,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "request_count", "type": "quantitative", "title": "Requests" } @@ -324,7 +322,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "request_count" ], "seriesColumn": "none", "showLegend": false, @@ -358,7 +356,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "model", "filterObjects": { "scope": "{{scopes.[API Keys]}}", @@ -377,7 +374,7 @@ { "type": "sum", "names": [ - "value" + "total_usage" ] } ] @@ -385,7 +382,7 @@ "sort": { "by": [ [ - "value_sum", + "total_usage_sum", "desc" ] ], @@ -393,7 +390,7 @@ }, "metadata": [ { - "name": "value_sum", + "name": "total_usage_sum", "shape": [ "currency", { @@ -421,7 +418,7 @@ "title": "Model" }, "y": { - "field": "value_sum", + "field": "total_usage_sum", "type": "quantitative", "title": "Spend ($)" } @@ -439,7 +436,7 @@ "data-stream-bar-chart": { "xAxisData": "group_uniqueValues", "yAxisData": [ - "value_sum" + "total_usage_sum" ], "xAxisGroup": "none", "xAxisLabel": "Model", diff --git a/plugins/OpenRouter/v1/defaultContent/model.dash.json b/plugins/OpenRouter/v1/defaultContent/model.dash.json index 4bd1d9b3..fd951324 100644 --- a/plugins/OpenRouter/v1/defaultContent/model.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/model.dash.json @@ -303,7 +303,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "filterObjects": { "scope": "{{scopes.[Models]}}", "workspace": "{{workspaceId}}", @@ -320,7 +319,7 @@ }, "metadata": [ { - "name": "value", + "name": "total_usage", "shape": [ "currency", { @@ -348,7 +347,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "total_usage", "type": "quantitative", "title": "Spend ($)" } @@ -365,7 +364,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "total_usage" ], "seriesColumn": "none", "showLegend": false, @@ -399,7 +398,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "request_count", "filterObjects": { "scope": "{{scopes.[Models]}}", "workspace": "{{workspaceId}}", @@ -416,7 +414,7 @@ }, "metadata": [ { - "name": "value", + "name": "request_count", "shape": [ "number", { @@ -444,7 +442,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "request_count", "type": "quantitative", "title": "Requests" } @@ -461,7 +459,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "request_count" ], "seriesColumn": "none", "showLegend": false, @@ -660,7 +658,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "provider", "filterObjects": { "scope": "{{scopes.[Models]}}", @@ -678,7 +675,7 @@ }, "metadata": [ { - "name": "value", + "name": "total_usage", "shape": [ "currency", { @@ -706,7 +703,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "total_usage", "type": "quantitative", "title": "Spend ($)" }, @@ -729,7 +726,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "total_usage" ], "seriesColumn": "group", "showLegend": true, @@ -764,7 +761,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "cost_per_million_tokens", "groupBy": "provider", "filterObjects": { "scope": "{{scopes.[Models]}}", @@ -782,7 +778,7 @@ }, "metadata": [ { - "name": "value", + "name": "cost_per_million_tokens", "shape": [ "currency", { @@ -810,7 +806,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "cost_per_million_tokens", "type": "quantitative", "title": "Cost per 1M tokens ($)" }, @@ -833,7 +829,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "cost_per_million_tokens" ], "seriesColumn": "group", "showLegend": true, @@ -864,11 +860,10 @@ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[usage]}}", - "name": "usage", + "id": "{{dataStreams.[performance]}}", + "name": "performance", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "p50_latency", "filterObjects": { "scope": "{{scopes.[Models]}}", "workspace": "{{workspaceId}}", @@ -885,7 +880,7 @@ }, "metadata": [ { - "name": "value", + "name": "p50_latency", "shape": "milliseconds" }, { @@ -907,7 +902,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "p50_latency", "type": "quantitative", "title": "Latency p50" } @@ -924,7 +919,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "p50_latency" ], "seriesColumn": "none", "showLegend": false, @@ -955,11 +950,10 @@ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[usage]}}", - "name": "usage", + "id": "{{dataStreams.[performance]}}", + "name": "performance", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "p50_total_time_to_first_token", "filterObjects": { "scope": "{{scopes.[Models]}}", "workspace": "{{workspaceId}}", @@ -976,7 +970,7 @@ }, "metadata": [ { - "name": "value", + "name": "p50_total_time_to_first_token", "shape": "milliseconds" }, { @@ -998,7 +992,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "p50_total_time_to_first_token", "type": "quantitative", "title": "Time To First Token p50" } @@ -1015,7 +1009,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "p50_total_time_to_first_token" ], "seriesColumn": "none", "showLegend": false, diff --git a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json index 751e280b..28a31dab 100644 --- a/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/modelComparison.dash.json @@ -1293,7 +1293,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "model", "filterObjects": { "scope": "{{scopes.[Model Comparison]}}", @@ -1311,7 +1310,7 @@ }, "metadata": [ { - "name": "value", + "name": "total_usage", "shape": [ "currency", { @@ -1334,7 +1333,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "total_usage" ], "seriesColumn": "group", "showLegend": true, @@ -1356,7 +1355,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "total_usage", "type": "quantitative", "title": "Spend ($)" }, @@ -1397,7 +1396,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "cost_per_million_tokens", "groupBy": "model", "filterObjects": { "scope": "{{scopes.[Model Comparison]}}", @@ -1415,7 +1413,7 @@ }, "metadata": [ { - "name": "value", + "name": "cost_per_million_tokens", "shape": [ "currency", { @@ -1438,7 +1436,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "cost_per_million_tokens" ], "seriesColumn": "group", "showLegend": true, @@ -1460,7 +1458,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "cost_per_million_tokens", "type": "quantitative", "title": "Cost per 1M tokens ($)" }, diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json index 4f4c4641..3bbdb5d2 100644 --- a/plugins/OpenRouter/v1/defaultContent/overview.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -104,23 +104,21 @@ "id": "{{dataStreams.[usage]}}", "name": "usage", "pluginConfigId": "{{configId}}", - "dataSourceConfig": { - "metric": "total_usage" - }, + "dataSourceConfig": {}, "group": { "by": [], "aggregate": [ { "type": "sum", "names": [ - "value" + "total_usage" ] } ] }, "metadata": [ { - "name": "value_sum", + "name": "total_usage_sum", "shape": [ "currency", { @@ -141,7 +139,7 @@ "encoding": { "values": [ { - "field": "value_sum", + "field": "total_usage_sum", "type": "quantitative", "title": "Spend (This Month)" } @@ -156,7 +154,7 @@ "type": "data-stream-scalar", "config": { "data-stream-scalar": { - "value": "value_sum", + "value": "total_usage_sum", "comparisonColumn": "none", "label": "Spend (This Month)" } @@ -185,23 +183,21 @@ "id": "{{dataStreams.[usage]}}", "name": "usage", "pluginConfigId": "{{configId}}", - "dataSourceConfig": { - "metric": "request_count" - }, + "dataSourceConfig": {}, "group": { "by": [], "aggregate": [ { "type": "sum", "names": [ - "value" + "request_count" ] } ] }, "metadata": [ { - "name": "value_sum", + "name": "request_count_sum", "shape": [ "number", { @@ -222,7 +218,7 @@ "encoding": { "values": [ { - "field": "value_sum", + "field": "request_count_sum", "type": "quantitative", "title": "Requests (This Month)" } @@ -237,7 +233,7 @@ "type": "data-stream-scalar", "config": { "data-stream-scalar": { - "value": "value_sum", + "value": "request_count_sum", "comparisonColumn": "none", "label": "Requests (This Month)" } @@ -265,9 +261,7 @@ "id": "{{dataStreams.[usage]}}", "name": "usage", "pluginConfigId": "{{configId}}", - "dataSourceConfig": { - "metric": "total_usage" - }, + "dataSourceConfig": {}, "sort": { "by": [ [ @@ -278,7 +272,7 @@ }, "metadata": [ { - "name": "value", + "name": "total_usage", "shape": [ "currency", { @@ -303,7 +297,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "total_usage", "type": "quantitative", "title": "Spend ($)" } @@ -320,7 +314,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "total_usage" ], "seriesColumn": "none", "showLegend": false, @@ -353,9 +347,7 @@ "id": "{{dataStreams.[usage]}}", "name": "usage", "pluginConfigId": "{{configId}}", - "dataSourceConfig": { - "metric": "request_count" - }, + "dataSourceConfig": {}, "sort": { "by": [ [ @@ -366,7 +358,7 @@ }, "metadata": [ { - "name": "value", + "name": "request_count", "shape": [ "number", { @@ -391,7 +383,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "request_count", "type": "quantitative", "title": "Requests" } @@ -408,7 +400,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "request_count" ], "seriesColumn": "none", "showLegend": false, @@ -442,7 +434,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "model" }, "sort": { @@ -455,7 +446,7 @@ }, "metadata": [ { - "name": "value", + "name": "total_usage", "shape": [ "currency", { @@ -480,7 +471,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "total_usage", "type": "quantitative", "aggregate": "sum", "title": "Spend ($)" @@ -503,7 +494,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "total_usage" ], "seriesColumn": "group", "showLegend": true, @@ -536,7 +527,7 @@ "id": "datastream-sql", "dataSourceConfig": { "version": "2.0", - "sql": "WITH by_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n m.id,\n -- Models missing from the models stream (e.g. stealth models) link to their OpenRouter page\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' ELSE 'https://openrouter.ai/' || u.slug END AS url,\n COALESCE(m.name, u.slug) AS name,\n u.slug,\n u.spend,\n u.requests,\n u.costPerRequest,\n u.promptTokens,\n u.completionTokens\nFROM Usage u\nLEFT JOIN by_slug m ON m.slug = u.slug\nORDER BY u.spend DESC", + "sql": "-- Usage arrives per model per time bucket; total each model over the timeframe\nWITH usage_by_model AS (\n SELECT\n \"group\" AS slug,\n sum(total_usage) AS spend,\n sum(request_count) AS requests,\n sum(tokens_prompt) AS promptTokens,\n sum(tokens_completion) AS completionTokens\n FROM Usage\n GROUP BY \"group\"\n),\nby_slug AS (\n SELECT\n canonical_slug AS slug,\n id,\n name\n FROM Models\n QUALIFY row_number() OVER (PARTITION BY canonical_slug ORDER BY length(id), id) = 1\n)\nSELECT\n m.id,\n -- Models missing from the models stream (e.g. stealth models) link to their OpenRouter page\n CASE WHEN m.id IS NOT NULL THEN '/drilldown/node?sourceId=Model%2F' || replace(replace(m.id, '/', '%2F'), ':', '%3A') || '&__configId={{configId}}' ELSE 'https://openrouter.ai/' || u.slug END AS url,\n COALESCE(m.name, u.slug) AS name,\n u.slug,\n u.spend,\n u.requests,\n u.spend / NULLIF(u.requests, 0) AS costPerRequest,\n u.promptTokens,\n u.completionTokens\nFROM usage_by_model u\nLEFT JOIN by_slug m ON m.slug = u.slug\nORDER BY u.spend DESC", "tables": [ { "tableName": "Usage", @@ -545,9 +536,12 @@ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[modelUsage]}}", - "name": "modelUsage", - "pluginConfigId": "{{configId}}" + "id": "{{dataStreams.[usage]}}", + "name": "usage", + "pluginConfigId": "{{configId}}", + "dataSourceConfig": { + "groupBy": "model" + } } } }, @@ -730,7 +724,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "model" }, "group": { @@ -744,7 +737,7 @@ { "type": "sum", "names": [ - "value" + "total_usage" ] } ] @@ -752,7 +745,7 @@ "sort": { "by": [ [ - "value_sum", + "total_usage_sum", "desc" ] ], @@ -760,7 +753,7 @@ }, "metadata": [ { - "name": "value_sum", + "name": "total_usage_sum", "shape": [ "currency", { @@ -785,7 +778,7 @@ "title": "Model" }, "y": { - "field": "value_sum", + "field": "total_usage_sum", "type": "quantitative", "title": "Spend ($)" } @@ -803,7 +796,7 @@ "data-stream-bar-chart": { "xAxisData": "group_uniqueValues", "yAxisData": [ - "value_sum" + "total_usage_sum" ], "xAxisGroup": "none", "xAxisLabel": "Model", @@ -847,7 +840,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "workspace" }, "group": { @@ -861,7 +853,7 @@ { "type": "sum", "names": [ - "value" + "total_usage" ] } ] @@ -869,7 +861,7 @@ "sort": { "by": [ [ - "value_sum", + "total_usage_sum", "desc" ] ], @@ -877,7 +869,7 @@ }, "metadata": [ { - "name": "value_sum", + "name": "total_usage_sum", "shape": [ "currency", { @@ -902,7 +894,7 @@ "title": "Workspace" }, "value": { - "field": "value_sum", + "field": "total_usage_sum", "type": "quantitative", "title": "Spend ($)" } @@ -918,7 +910,7 @@ "type": "data-stream-donut-chart", "config": { "data-stream-donut-chart": { - "valueColumn": "value_sum", + "valueColumn": "total_usage_sum", "labelColumn": "group_uniqueValues", "hideCenterValue": false, "showValuesAsPercentage": true, @@ -951,7 +943,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "provider" }, "group": { @@ -965,7 +956,7 @@ { "type": "sum", "names": [ - "value" + "total_usage" ] } ] @@ -973,14 +964,14 @@ "sort": { "by": [ [ - "value_sum", + "total_usage_sum", "desc" ] ] }, "metadata": [ { - "name": "value_sum", + "name": "total_usage_sum", "shape": [ "currency", { @@ -1005,7 +996,7 @@ "title": "Provider" }, "value": { - "field": "value_sum", + "field": "total_usage_sum", "type": "quantitative", "title": "Spend ($)" } @@ -1021,7 +1012,7 @@ "type": "data-stream-donut-chart", "config": { "data-stream-donut-chart": { - "valueColumn": "value_sum", + "valueColumn": "total_usage_sum", "labelColumn": "group_uniqueValues", "hideCenterValue": false, "showValuesAsPercentage": true, @@ -1050,11 +1041,10 @@ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[usage]}}", - "name": "usage", + "id": "{{dataStreams.[performance]}}", + "name": "performance", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "p50_latency", "groupBy": "model" }, "sort": { @@ -1067,7 +1057,7 @@ }, "metadata": [ { - "name": "value", + "name": "p50_latency", "shape": "milliseconds" }, { @@ -1086,7 +1076,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "p50_latency", "type": "quantitative", "title": "Latency p50" }, @@ -1107,7 +1097,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "p50_latency" ], "seriesColumn": "group", "showLegend": true, diff --git a/plugins/OpenRouter/v1/defaultContent/provider.dash.json b/plugins/OpenRouter/v1/defaultContent/provider.dash.json index 4242661b..776f8f48 100644 --- a/plugins/OpenRouter/v1/defaultContent/provider.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/provider.dash.json @@ -125,7 +125,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "filterObjects": { "scope": "{{scopes.[Providers]}}", "workspace": "{{workspaceId}}", @@ -142,7 +141,7 @@ }, "metadata": [ { - "name": "value", + "name": "total_usage", "shape": [ "currency", { @@ -170,7 +169,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "total_usage", "type": "quantitative", "title": "Spend ($)" } @@ -187,7 +186,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "total_usage" ], "seriesColumn": "none", "showLegend": false, @@ -222,7 +221,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "request_count", "filterObjects": { "scope": "{{scopes.[Providers]}}", "workspace": "{{workspaceId}}", @@ -239,7 +237,7 @@ }, "metadata": [ { - "name": "value", + "name": "request_count", "shape": [ "number", { @@ -267,7 +265,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "request_count", "type": "quantitative", "title": "Requests" } @@ -284,7 +282,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "request_count" ], "seriesColumn": "none", "showLegend": false, @@ -315,11 +313,10 @@ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[usage]}}", - "name": "usage", + "id": "{{dataStreams.[performance]}}", + "name": "performance", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "p50_latency", "filterObjects": { "scope": "{{scopes.[Providers]}}", "workspace": "{{workspaceId}}", @@ -336,7 +333,7 @@ }, "metadata": [ { - "name": "value", + "name": "p50_latency", "shape": "milliseconds" }, { @@ -358,7 +355,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "p50_latency", "type": "quantitative", "title": "Latency p50" } @@ -375,7 +372,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "p50_latency" ], "seriesColumn": "none", "showLegend": false, @@ -406,11 +403,10 @@ "{{configId}}" ], "dataStream": { - "id": "{{dataStreams.[usage]}}", - "name": "usage", + "id": "{{dataStreams.[performance]}}", + "name": "performance", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "p50_throughput", "filterObjects": { "scope": "{{scopes.[Providers]}}", "workspace": "{{workspaceId}}", @@ -427,7 +423,7 @@ }, "metadata": [ { - "name": "value", + "name": "p50_throughput", "shape": [ "number", { @@ -454,7 +450,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "p50_throughput", "type": "quantitative", "title": "Tokens/s" } @@ -471,7 +467,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "p50_throughput" ], "seriesColumn": "none", "showLegend": false, @@ -506,7 +502,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "model", "filterObjects": { "scope": "{{scopes.[Providers]}}", @@ -525,7 +520,7 @@ { "type": "sum", "names": [ - "value" + "total_usage" ] } ] @@ -533,7 +528,7 @@ "sort": { "by": [ [ - "value_sum", + "total_usage_sum", "desc" ] ], @@ -541,7 +536,7 @@ }, "metadata": [ { - "name": "value_sum", + "name": "total_usage_sum", "shape": [ "currency", { @@ -569,7 +564,7 @@ "title": "Model" }, "y": { - "field": "value_sum", + "field": "total_usage_sum", "type": "quantitative", "title": "Spend ($)" } @@ -587,7 +582,7 @@ "data-stream-bar-chart": { "xAxisData": "group_uniqueValues", "yAxisData": [ - "value_sum" + "total_usage_sum" ], "xAxisGroup": "none", "xAxisLabel": "Model", diff --git a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json index ff3b4fe0..769838d3 100644 --- a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json @@ -195,7 +195,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "filterObjects": { "scope": "{{scopes.[Workspaces]}}", "workspace": "{{workspaceId}}", @@ -212,7 +211,7 @@ }, "metadata": [ { - "name": "value", + "name": "total_usage", "shape": [ "currency", { @@ -240,7 +239,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "total_usage", "type": "quantitative", "title": "Spend ($)" } @@ -257,7 +256,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "total_usage" ], "seriesColumn": "none", "showLegend": false, @@ -291,7 +290,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "request_count", "filterObjects": { "scope": "{{scopes.[Workspaces]}}", "workspace": "{{workspaceId}}", @@ -308,7 +306,7 @@ }, "metadata": [ { - "name": "value", + "name": "request_count", "shape": [ "number", { @@ -336,7 +334,7 @@ "title": "Date" }, "y": { - "field": "value", + "field": "request_count", "type": "quantitative", "title": "Requests" } @@ -353,7 +351,7 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "value" + "request_count" ], "seriesColumn": "none", "showLegend": false, @@ -387,7 +385,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "model", "filterObjects": { "scope": "{{scopes.[Workspaces]}}", @@ -406,7 +403,7 @@ { "type": "sum", "names": [ - "value" + "total_usage" ] } ] @@ -414,7 +411,7 @@ "sort": { "by": [ [ - "value_sum", + "total_usage_sum", "desc" ] ], @@ -422,7 +419,7 @@ }, "metadata": [ { - "name": "value_sum", + "name": "total_usage_sum", "shape": [ "currency", { @@ -450,7 +447,7 @@ "title": "Model" }, "y": { - "field": "value_sum", + "field": "total_usage_sum", "type": "quantitative", "title": "Spend ($)" } @@ -468,7 +465,7 @@ "data-stream-bar-chart": { "xAxisData": "group_uniqueValues", "yAxisData": [ - "value_sum" + "total_usage_sum" ], "xAxisGroup": "none", "xAxisLabel": "Model", @@ -512,7 +509,6 @@ "name": "usage", "pluginConfigId": "{{configId}}", "dataSourceConfig": { - "metric": "total_usage", "groupBy": "api_key_id", "filterObjects": { "scope": "{{scopes.[Workspaces]}}", @@ -531,7 +527,7 @@ { "type": "sum", "names": [ - "value" + "total_usage" ] } ] @@ -539,7 +535,7 @@ "sort": { "by": [ [ - "value_sum", + "total_usage_sum", "desc" ] ], @@ -547,7 +543,7 @@ }, "metadata": [ { - "name": "value_sum", + "name": "total_usage_sum", "shape": [ "currency", { @@ -575,7 +571,7 @@ "title": "API Key" }, "value": { - "field": "value_sum", + "field": "total_usage_sum", "type": "quantitative", "title": "Spend ($)" } @@ -591,7 +587,7 @@ "type": "data-stream-donut-chart", "config": { "data-stream-donut-chart": { - "valueColumn": "value_sum", + "valueColumn": "total_usage_sum", "labelColumn": "group_uniqueValues", "hideCenterValue": false, "showValuesAsPercentage": true, diff --git a/plugins/OpenRouter/v1/docs/README.md b/plugins/OpenRouter/v1/docs/README.md index b9c878f8..e6782bca 100644 --- a/plugins/OpenRouter/v1/docs/README.md +++ b/plugins/OpenRouter/v1/docs/README.md @@ -38,8 +38,8 @@ The out-of-the-box dashboards include an account **Overview**; **Model Leaderboa ## Data streams -- **Usage** — spend, cost per 1M tokens, requests, tokens, cache hit rate, latency, time to first token, or throughput over time, optionally grouped by model, provider, workspace, API key, app, user, origin, country, finish reason, or streamed, and optionally filtered to selected models, providers, workspaces, or API keys. Any other OpenRouter analytics metric or dimension can be typed in. Account-wide. Management key only. -- **Model Usage** — spend, requests, cost per request, and prompt and completion tokens for every model over the timeframe, keyed by canonical slug. OpenRouter reports `:free` and `:batch` variants under the same slug as their base model, so their usage is combined. The overview dashboard joins this to the imported models for names and drilldown. Account-wide. Management key only. +- **Usage** — spend, BYOK spend, requests, and total, prompt, completion, reasoning, and cached tokens over time, with cost per 1M tokens, optionally grouped by model, provider, workspace, API key, app, user, origin, country, finish reason, or streamed, and optionally filtered to selected models, providers, workspaces, or API keys. Any other OpenRouter analytics dimension can be typed in. Usage is reported by canonical slug, with `:free` and `:batch` variants combined with their base model. Account-wide. Management key only. +- **Performance** — latency and time to first token (p50 and p90) and throughput (p50) over time, with the same grouping and filtering as **Usage**. The last 30 days at most. Account-wide. Management key only. - **API Keys** — API keys in a workspace with spend and limits. Per workspace. Management key only. - **Workspaces** — workspaces in the account with their default models, provider sort, disabled server tools, whether BYOK spend counts toward budgets, and logging and broadcast settings, optionally filtered to selected workspaces. Account-wide. Management key only. - **Credits** — credits purchased, used, and remaining. Account-wide. @@ -70,7 +70,7 @@ With a standard API key, only models and providers are imported; the workspace a - **Benchmarks, rankings, app rankings, task classifications, and session costs refresh at most once a day** — OpenRouter limits these endpoints to 500 requests a day per account, so the plugin caches them for 24 hours. - **Rankings licence** — rankings, app ranking, task classification, session cost, and benchmark data is published by OpenRouter under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). Each row of these streams has a hidden **Citation** column with the exact attribution OpenRouter requires, and a hidden **As Of** date. The **Model Leaderboards**, **Model Comparison**, **Ecosystem**, and **Model** dashboards credit the data already; show the citation on any dashboard you build from it. - **Not every model has benchmark scores** — Artificial Analysis scores cover fewer than half of models; the rest show blank. -- **Usage history** can be queried up to a year back. Latency, time to first token, throughput, and grouping by **Provider**, **Origin**, **Country**, **Finish reason**, or **Streamed** are limited to 31 days; longer timeframes return an error. +- **Usage history** can be queried up to a year back. Grouping by **Provider**, **Origin**, **Country**, **Finish reason**, or **Streamed** is limited to 31 days; longer timeframes return an error. - **Usage timeframes start at 12 hours** — data is bucketed by hour, day, week, or month to suit the range; periods with no activity are left out rather than shown as zero. - **Cost per 1M tokens is blended** — it's spend divided by all tokens, including cached and reasoning tokens, so it reflects your mix of traffic rather than a provider's list price. - **Large usage queries may be truncated** — the stream warns when this happens; narrow the timeframe or filter to specific objects. From bbc1b24787aa04d969e54ccb0bad91b60b9ab6ef Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 17:29:38 +0100 Subject: [PATCH 12/16] Show p90 latency and fill the usage lines on OpenRouter dashboards Your Latency and Your Time To First Token on the Model and Provider dashboards now draw p50 as a line with a shaded band up to p90, so the tail shows without a second line. The single-series spend and requests lines get an area fill, and Latency p50 By Model gets the y-axis title its sibling charts have. Co-Authored-By: Claude Opus 5.5 --- .../v1/defaultContent/apiKey.dash.json | 6 +- .../v1/defaultContent/model.dash.json | 56 +++++++++++++------ .../v1/defaultContent/overview.dash.json | 10 +++- .../v1/defaultContent/provider.dash.json | 31 +++++++--- .../v1/defaultContent/workspace.dash.json | 6 +- 5 files changed, 77 insertions(+), 32 deletions(-) diff --git a/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json index d19dac53..5bf6c3f8 100644 --- a/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json @@ -217,7 +217,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Spend ($)" + "cartesianYAxisTitle": "Spend ($)", + "cartesianAreaGradient": true } } }, @@ -312,7 +313,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Requests" + "cartesianYAxisTitle": "Requests", + "cartesianAreaGradient": true } } }, diff --git a/plugins/OpenRouter/v1/defaultContent/model.dash.json b/plugins/OpenRouter/v1/defaultContent/model.dash.json index fd951324..12516b31 100644 --- a/plugins/OpenRouter/v1/defaultContent/model.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/model.dash.json @@ -354,7 +354,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Spend ($)" + "cartesianYAxisTitle": "Spend ($)", + "cartesianAreaGradient": true } } }, @@ -449,7 +450,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Requests" + "cartesianYAxisTitle": "Requests", + "cartesianAreaGradient": true } } }, @@ -854,8 +856,8 @@ "z": 0, "config": { "_type": "tile/data-stream", - "title": "Your Latency p50 (Last 30 Days)", - "description": "", + "title": "Your Latency (Last 30 Days)", + "description": "Line: your median (p50) latency each day. Shaded band: p50 to p90, so 9 in 10 requests were faster than its top edge.", "activePluginConfigIds": [ "{{configId}}" ], @@ -904,12 +906,22 @@ "y": { "field": "p50_latency", "type": "quantitative", - "title": "Latency p50" + "title": "p50" + }, + "low": { + "field": "p50_latency", + "type": "quantitative", + "title": "p50" + }, + "high": { + "field": "p90_latency", + "type": "quantitative", + "title": "p90" } }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Latency p50" + "cartesianYAxisTitle": "Latency" } } }, @@ -919,12 +931,13 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "p50_latency" + "p50_latency", + "p90_latency" ], "seriesColumn": "none", - "showLegend": false, + "showLegend": true, "legendPosition": "bottom", - "yAxisLabel": "Latency p50", + "yAxisLabel": "Latency", "showYAxisLabel": true, "showTrendLine": false } @@ -944,8 +957,8 @@ "z": 0, "config": { "_type": "tile/data-stream", - "title": "Your Time To First Token p50 (Last 30 Days)", - "description": "", + "title": "Your Time To First Token (Last 30 Days)", + "description": "Line: your median (p50) time to first token each day. Shaded band: p50 to p90, so 9 in 10 requests were faster than its top edge.", "activePluginConfigIds": [ "{{configId}}" ], @@ -994,12 +1007,22 @@ "y": { "field": "p50_total_time_to_first_token", "type": "quantitative", - "title": "Time To First Token p50" + "title": "p50" + }, + "low": { + "field": "p50_total_time_to_first_token", + "type": "quantitative", + "title": "p50" + }, + "high": { + "field": "p90_total_time_to_first_token", + "type": "quantitative", + "title": "p90" } }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Time To First Token p50" + "cartesianYAxisTitle": "Time To First Token" } } }, @@ -1009,12 +1032,13 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "p50_total_time_to_first_token" + "p50_total_time_to_first_token", + "p90_total_time_to_first_token" ], "seriesColumn": "none", - "showLegend": false, + "showLegend": true, "legendPosition": "bottom", - "yAxisLabel": "Time To First Token p50", + "yAxisLabel": "Time To First Token", "showYAxisLabel": true, "showTrendLine": false } diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json index 3bbdb5d2..a6db5d42 100644 --- a/plugins/OpenRouter/v1/defaultContent/overview.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -304,7 +304,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Spend ($)" + "cartesianYAxisTitle": "Spend ($)", + "cartesianAreaGradient": true } } }, @@ -390,7 +391,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Requests" + "cartesianYAxisTitle": "Requests", + "cartesianAreaGradient": true } } }, @@ -1087,7 +1089,9 @@ } }, "style": { - "legendPosition": "bottom" + "legendPosition": "bottom", + "cartesianShowYAxisTitle": true, + "cartesianYAxisTitle": "Latency p50" } } }, diff --git a/plugins/OpenRouter/v1/defaultContent/provider.dash.json b/plugins/OpenRouter/v1/defaultContent/provider.dash.json index 776f8f48..6980518c 100644 --- a/plugins/OpenRouter/v1/defaultContent/provider.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/provider.dash.json @@ -176,7 +176,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Spend ($)" + "cartesianYAxisTitle": "Spend ($)", + "cartesianAreaGradient": true } } }, @@ -272,7 +273,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Requests" + "cartesianYAxisTitle": "Requests", + "cartesianAreaGradient": true } } }, @@ -307,8 +309,8 @@ "z": 0, "config": { "_type": "tile/data-stream", - "title": "Your Latency p50 (Last 30 Days)", - "description": "", + "title": "Your Latency (Last 30 Days)", + "description": "Line: your median (p50) latency each day. Shaded band: p50 to p90, so 9 in 10 requests were faster than its top edge.", "activePluginConfigIds": [ "{{configId}}" ], @@ -357,12 +359,22 @@ "y": { "field": "p50_latency", "type": "quantitative", - "title": "Latency p50" + "title": "p50" + }, + "low": { + "field": "p50_latency", + "type": "quantitative", + "title": "p50" + }, + "high": { + "field": "p90_latency", + "type": "quantitative", + "title": "p90" } }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Latency p50" + "cartesianYAxisTitle": "Latency" } } }, @@ -372,12 +384,13 @@ "data-stream-line-graph": { "xAxisColumn": "date", "yAxisColumn": [ - "p50_latency" + "p50_latency", + "p90_latency" ], "seriesColumn": "none", - "showLegend": false, + "showLegend": true, "legendPosition": "bottom", - "yAxisLabel": "Latency p50", + "yAxisLabel": "Latency", "showYAxisLabel": true, "showTrendLine": false } diff --git a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json index 769838d3..55b232b2 100644 --- a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json @@ -246,7 +246,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Spend ($)" + "cartesianYAxisTitle": "Spend ($)", + "cartesianAreaGradient": true } } }, @@ -341,7 +342,8 @@ }, "style": { "cartesianShowYAxisTitle": true, - "cartesianYAxisTitle": "Requests" + "cartesianYAxisTitle": "Requests", + "cartesianAreaGradient": true } } }, From 7c98a32ac7a7c0bd2c4bb4f629dc84cbef297b89 Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 17:29:38 +0100 Subject: [PATCH 13/16] Show how much of each OpenRouter API key's limit is used The apiKeys stream gains a computed Limit Used column: the share of a key's limit spent in its current reset period, empty for keys without a limit. The API Keys tables on Overview and Workspace show it as a progress bar beside Limit and Limit Remaining, tinting the row amber above 80% and red above 95%. The API Key perspective gets a Spend Limit gauge for its own key, with the same breakpoints. The apiKeys stream lists keys per workspace, so the tile's SQL picks the key out by joining its hash to the page object's rawId. Co-Authored-By: Claude Opus 5.5 --- .../OpenRouter/v1/dataStreams/apiKeys.json | 7 + .../v1/defaultContent/apiKey.dash.json | 139 +++++++++++++++++- .../v1/defaultContent/overview.dash.json | 34 ++++- .../v1/defaultContent/workspace.dash.json | 34 ++++- 4 files changed, 206 insertions(+), 8 deletions(-) diff --git a/plugins/OpenRouter/v1/dataStreams/apiKeys.json b/plugins/OpenRouter/v1/dataStreams/apiKeys.json index 4496bf21..16429c04 100644 --- a/plugins/OpenRouter/v1/dataStreams/apiKeys.json +++ b/plugins/OpenRouter/v1/dataStreams/apiKeys.json @@ -73,6 +73,13 @@ "displayName": "Limit Remaining", "shape": "usd" }, + { + "name": "limitUsed", + "displayName": "Limit Used (%)", + "shape": ["number", { "decimalPlaces": 1 }], + "computed": true, + "valueExpression": "{{ $['limit'] > 0 && $['limit_remaining'] != null ? Math.min(100, 100 * ($['limit'] - $['limit_remaining']) / $['limit']) : null }}" + }, { "name": "limit_reset", "displayName": "Limit Reset", diff --git a/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json index 5bf6c3f8..fc495938 100644 --- a/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/apiKey.dash.json @@ -339,9 +339,9 @@ }, { "i": "5c1cfeba-46df-41a0-aebc-9fcfb2a35a81", - "x": 0, + "x": 1, "y": 4, - "w": 4, + "w": 3, "h": 3, "moved": false, "static": false, @@ -460,6 +460,141 @@ } } } + }, + { + "i": "7e935a7c-91fa-4ead-a346-f440f059bff3", + "x": 0, + "y": 4, + "w": 1, + "h": 3, + "moved": false, + "static": false, + "z": 0, + "config": { + "_type": "tile/data-stream", + "title": "Spend Limit", + "description": "", + "timeframe": "none", + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "datastream-sql", + "dataSourceConfig": { + "version": "2.0", + "sql": "-- apiKeys is listed per workspace, so pick this page's key out by its hash;\n-- keys without a limit return no row\nSELECT\n k.name,\n k.limitUsed\nFROM Keys k\nJOIN ThisKey p ON p.rawId = k.hash\nWHERE k.limitUsed IS NOT NULL", + "tables": [ + { + "tableName": "Keys", + "config": { + "activePluginConfigIds": [ + "{{configId}}" + ], + "dataStream": { + "id": "{{dataStreams.[apiKeys]}}", + "name": "apiKeys", + "pluginConfigId": "{{configId}}" + }, + "scope": { + "scope": "{{scopes.[Workspaces]}}", + "workspace": "{{workspaceId}}" + }, + "timeframe": "none" + } + }, + { + "tableName": "ThisKey", + "config": { + "dataStream": { + "id": "datastream-properties", + "name": "properties" + }, + "scope": { + "scope": "{{scopes.[API Keys]}}", + "workspace": "{{workspaceId}}", + "variable": "{{variables.[API Key]}}" + }, + "variables": [ + "{{variables.[API Key]}}" + ], + "timeframe": "none" + } + } + ] + }, + "metadata": [ + { + "name": "name", + "displayName": "API Key", + "shape": "string", + "role": "label" + }, + { + "name": "limitUsed", + "displayName": "Limit Used (%)", + "shape": [ + "number", + { + "decimalPlaces": 1 + } + ], + "role": "value" + } + ] + }, + "variables": [ + "{{variables.[API Key]}}" + ], + "vizSpec": { + "version": "1.0", + "visualization": { + "chartType": "gauge", + "encoding": { + "value": { + "field": "limitUsed", + "type": "quantitative", + "title": "Limit Used (%)" + } + }, + "style": { + "gaugeMin": 0, + "gaugeMax": 100, + "gaugeLabelText": "Limit Used", + "labelFormat": "{{{{raw}}}}{{value}}%{{{{/raw}}}}", + "gaugeBands": [ + { + "to": 0.8, + "color": "#91cc75" + }, + { + "to": 0.95, + "color": "#fac858" + }, + { + "to": 1, + "color": "#ee6666" + } + ] + } + } + }, + "visualisation": { + "type": "data-stream-gauge", + "config": { + "data-stream-gauge": { + "value": { + "type": "mean", + "columns": [ + "limitUsed" + ] + }, + "label": "Limit Used (%)", + "minimum": 0, + "maximum": 100 + } + } + } + } } ] } diff --git a/plugins/OpenRouter/v1/defaultContent/overview.dash.json b/plugins/OpenRouter/v1/defaultContent/overview.dash.json index a6db5d42..42a28668 100644 --- a/plugins/OpenRouter/v1/defaultContent/overview.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/overview.dash.json @@ -1143,7 +1143,17 @@ ] ], "top": 10 - } + }, + "metadata": [ + { + "name": "limitStatus", + "displayName": "Limit Status", + "shape": "string", + "visible": false, + "computed": true, + "valueExpression": "{{{{raw}}}}{{ (u => u > 95 ? 'error' : u > 80 ? 'warning' : '')($['limit'] > 0 && $['limit_remaining'] != null ? 100 * ($['limit'] - $['limit_remaining']) / $['limit'] : 0) }}{{{{/raw}}}}" + } + ] }, "scope": { "scope": "{{scopes.[Workspaces]}}", @@ -1210,6 +1220,14 @@ "type": "text" } }, + { + "field": "limitUsed", + "title": "Limit Used", + "renderer": { + "type": "progress", + "max": 100 + } + }, { "field": "limit_reset", "title": "Limit Reset", @@ -1228,7 +1246,15 @@ } ] }, - "style": {} + "style": { + "richTableRowHighlight": { + "field": "limitStatus", + "colors": { + "warning": "rgba(245,158,11,0.15)", + "error": "rgba(239,68,68,0.15)" + } + } + } } }, "visualisation": { @@ -1244,6 +1270,7 @@ "usage", "limit", "limit_remaining", + "limitUsed", "limit_reset", "disabled" ], @@ -1251,7 +1278,8 @@ "hash", "workspace_id", "created_at", - "expires_at" + "expires_at", + "limitStatus" ] } } diff --git a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json index 55b232b2..77b83fb6 100644 --- a/plugins/OpenRouter/v1/defaultContent/workspace.dash.json +++ b/plugins/OpenRouter/v1/defaultContent/workspace.dash.json @@ -628,7 +628,17 @@ "desc" ] ] - } + }, + "metadata": [ + { + "name": "limitStatus", + "displayName": "Limit Status", + "shape": "string", + "visible": false, + "computed": true, + "valueExpression": "{{{{raw}}}}{{ (u => u > 95 ? 'error' : u > 80 ? 'warning' : '')($['limit'] > 0 && $['limit_remaining'] != null ? 100 * ($['limit'] - $['limit_remaining']) / $['limit'] : 0) }}{{{{/raw}}}}" + } + ] }, "scope": { "scope": "{{scopes.[Workspaces]}}", @@ -699,6 +709,14 @@ "type": "text" } }, + { + "field": "limitUsed", + "title": "Limit Used", + "renderer": { + "type": "progress", + "max": 100 + } + }, { "field": "limit_reset", "title": "Limit Reset", @@ -717,7 +735,15 @@ } ] }, - "style": {} + "style": { + "richTableRowHighlight": { + "field": "limitStatus", + "colors": { + "warning": "rgba(245,158,11,0.15)", + "error": "rgba(239,68,68,0.15)" + } + } + } } }, "visualisation": { @@ -733,6 +759,7 @@ "usage", "limit", "limit_remaining", + "limitUsed", "limit_reset", "disabled" ], @@ -740,7 +767,8 @@ "hash", "workspace_id", "created_at", - "expires_at" + "expires_at", + "limitStatus" ] } } From d9d0aa75f157dcdca13ddc6cf85128c2062f81ee Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 19:32:56 +0100 Subject: [PATCH 14/16] List everything a standard OpenRouter key can and can't read Live testing with a standard key showed providers, model providers, app rankings, session cost and task classifications work too, and that performance needs a management key like usage does. The README, config form help and management-check warning now say so. Co-Authored-By: Claude Opus 5.5 --- plugins/OpenRouter/v1/configValidation.json | 2 +- plugins/OpenRouter/v1/docs/README.md | 10 +++++----- plugins/OpenRouter/v1/ui.json | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/OpenRouter/v1/configValidation.json b/plugins/OpenRouter/v1/configValidation.json index eb07eb2b..f24b2554 100644 --- a/plugins/OpenRouter/v1/configValidation.json +++ b/plugins/OpenRouter/v1/configValidation.json @@ -16,7 +16,7 @@ }, "required": false, "success": "Management key detected. All data streams are available.", - "error": "Could not read workspaces. Workspaces, API keys, and usage need a management key; model, benchmark, ranking, and credit data still work." + "error": "Could not read workspaces. Workspaces, API keys, usage, and performance need a management key; everything else still works." } ] } diff --git a/plugins/OpenRouter/v1/docs/README.md b/plugins/OpenRouter/v1/docs/README.md index e6782bca..398fccac 100644 --- a/plugins/OpenRouter/v1/docs/README.md +++ b/plugins/OpenRouter/v1/docs/README.md @@ -2,12 +2,12 @@ Monitor your [OpenRouter](https://openrouter.ai) spend, usage, and API keys in S ## Setup -You will need an OpenRouter **management key** for full access. A standard API key also works, but only for model, benchmark, ranking, and credit data. +You will need an OpenRouter **management key** for full access. A standard API key also works, but without workspace, API key, usage, and performance data. | Key type | What works | | -------- | ---------- | -| **Management key** | Everything: models, benchmarks, rankings, credits, workspaces, API keys, and usage. | -| **Standard API key** | Models, benchmarks, rankings, and credits. Workspaces, API keys, and usage are unavailable. | +| **Management key** | Everything. | +| **Standard API key** | Models, providers, model providers, benchmarks, rankings, app rankings, session cost, task classifications, and credits. Workspaces, API keys, usage, and performance are unavailable. | Management keys can read and manage account, workspace, API key, and usage data, but cannot be used to call models. This plugin only reads data; it never creates, modifies, or deletes anything in OpenRouter. @@ -22,7 +22,7 @@ Management keys can read and manage account, workspace, API key, and usage data, | ----------- | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | -------- | | **API key** | A management key (recommended) or standard API key, sent as a Bearer token. | OpenRouter → **Settings → Management Keys** (or **Settings → API Keys**). | Yes | -On save, the plugin checks the key in two steps. An invalid, disabled, or expired key fails setup. A valid standard API key passes with a warning that workspaces, API keys, and usage need a management key. +On save, the plugin checks the key in two steps. An invalid, disabled, or expired key fails setup. A valid standard API key passes with a warning that workspaces, API keys, usage, and performance need a management key. ## What this plugin monitors @@ -34,7 +34,7 @@ On save, the plugin checks the key in two steps. An invalid, disabled, or expire - **Provider performance** — current status, price, uptime, latency, and throughput for each provider serving a model. - **Benchmarks and rankings** — scores from Artificial Analysis, Design Arena, and OpenRouter's own evals, OpenRouter's public leaderboard of the most used models, what share of OpenRouter traffic goes to each kind of task, what a coding-agent session typically costs, and which public apps use OpenRouter most. -The out-of-the-box dashboards include an account **Overview**; **Model Leaderboards** and **Ecosystem** dashboards covering models, apps, task types, and providers across all of OpenRouter, which work with a standard API key; a **Model Comparison** dashboard that compares any models you pick side by side on price, benchmarks, providers, session cost, and your own usage (the usage tiles need a management key); and a perspective for each **Model**, **Provider**, **Workspace**, and **API Key** (spend tiles need a management key). The Model perspective includes a current view of each provider's status, price, and throughput, and your spend and cost per 1M tokens on each provider. +The out-of-the-box dashboards include an account **Overview**; **Model Leaderboards** and **Ecosystem** dashboards covering models, apps, task types, and providers across all of OpenRouter, which work with a standard API key; a **Model Comparison** dashboard that compares any models you pick side by side on price, benchmarks, providers, session cost, and your own usage (the usage tiles need a management key); and a perspective for each **Model**, **Provider**, **Workspace**, and **API Key** (spend and latency tiles need a management key). The Model perspective includes a current view of each provider's status, price, and throughput, and your spend and cost per 1M tokens on each provider. ## Data streams diff --git a/plugins/OpenRouter/v1/ui.json b/plugins/OpenRouter/v1/ui.json index 23698165..ad411ae0 100644 --- a/plugins/OpenRouter/v1/ui.json +++ b/plugins/OpenRouter/v1/ui.json @@ -7,6 +7,6 @@ "validation": { "required": true }, - "help": "Use a [management key](https://openrouter.ai/settings/management-keys) for full access to workspace, API key, and usage data. A standard [API key](https://openrouter.ai/settings/keys) covers model, benchmark, ranking, and credit data only. This plugin only reads data; it does not create, modify, or delete anything in OpenRouter." + "help": "Use a [management key](https://openrouter.ai/settings/management-keys) for full access. A standard [API key](https://openrouter.ai/settings/keys) works for everything except workspace, API key, usage, and performance data. This plugin only reads data; it does not create, modify, or delete anything in OpenRouter." } ] From 8481c8cb3e06f07bb8c80c195f11eb422fe4e757 Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Wed, 7 Oct 2026 19:36:50 +0100 Subject: [PATCH 15/16] Fall back to the model ID when OpenRouter omits a canonical slug Every model has a canonical slug today, but without one the permaslug became the string "undefined", which breaks the :free joins and makes the leaderboard dedupe collapse all such models into one row. Co-Authored-By: Claude Opus 5.5 --- plugins/OpenRouter/v1/dataStreams/models.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/OpenRouter/v1/dataStreams/models.json b/plugins/OpenRouter/v1/dataStreams/models.json index 71234c4c..38b87b46 100644 --- a/plugins/OpenRouter/v1/dataStreams/models.json +++ b/plugins/OpenRouter/v1/dataStreams/models.json @@ -52,7 +52,7 @@ "displayName": "Permaslug", "shape": "string", "computed": true, - "valueExpression": "{{ $['canonical_slug'] + ($['id'].endsWith(':free') ? ':free' : '') }}" + "valueExpression": "{{ ($['canonical_slug'] || $['id'].replace(/:free$/, '')) + ($['id'].endsWith(':free') ? ':free' : '') }}" }, { "name": "context_length", From 82c0a9ce17530241069c6550de8ede800d4c68f3 Mon Sep 17 00:00:00 2001 From: Dave Lewis Date: Thu, 8 Oct 2026 17:04:42 +0100 Subject: [PATCH 16/16] Make OpenRouter dataset filters single-select and clearable Autocompletes default to multiselect, so the dashboard sent arrays the OpenRouter dataset endpoints reject (e.g. subcategory=productivity, app_slug as an array). Category, Segment and Harness are now single-select, with typeof guards so empty values drop out of the URL. Session length and Benchmark source become clearable autocompletes, as leaving them empty returns every value. Model Rankings shows Day as the default period, matching what the API does when none is sent. Co-Authored-By: Claude Opus 5.5 --- .../v1/dataStreams/appRankings.json | 5 ++- .../OpenRouter/v1/dataStreams/benchmarks.json | 35 ++++++++------- .../OpenRouter/v1/dataStreams/rankings.json | 12 ++--- .../v1/dataStreams/sessionCost.json | 44 +++++++++++-------- 4 files changed, 55 insertions(+), 41 deletions(-) diff --git a/plugins/OpenRouter/v1/dataStreams/appRankings.json b/plugins/OpenRouter/v1/dataStreams/appRankings.json index 0fb54bbd..15c4a75b 100644 --- a/plugins/OpenRouter/v1/dataStreams/appRankings.json +++ b/plugins/OpenRouter/v1/dataStreams/appRankings.json @@ -27,11 +27,11 @@ }, { "key": "category", - "value": "{{ (c => ['coding', 'creative', 'productivity', 'entertainment'].includes(c) ? c : null)(typeof category !== 'undefined' ? category : null) }}" + "value": "{{ typeof category === 'string' && ['coding', 'creative', 'productivity', 'entertainment'].includes(category) ? category : null }}" }, { "key": "subcategory", - "value": "{{ (c => c && !['coding', 'creative', 'productivity', 'entertainment'].includes(c) ? c : null)(typeof category !== 'undefined' ? category : null) }}" + "value": "{{ typeof category === 'string' && category && !['coding', 'creative', 'productivity', 'entertainment'].includes(category) ? category : null }}" }, { "key": "limit", @@ -131,6 +131,7 @@ "type": "autocomplete", "name": "category", "label": "Category", + "isMulti": false, "isClearable": true, "data": { "source": "fixed", diff --git a/plugins/OpenRouter/v1/dataStreams/benchmarks.json b/plugins/OpenRouter/v1/dataStreams/benchmarks.json index ba4a5d5f..71575f96 100644 --- a/plugins/OpenRouter/v1/dataStreams/benchmarks.json +++ b/plugins/OpenRouter/v1/dataStreams/benchmarks.json @@ -25,23 +25,28 @@ "matches": "none", "ui": [ { - "type": "choiceChips", + "type": "autocomplete", "name": "source", "label": "Source", - "options": [ - { - "value": "artificial-analysis", - "label": "Artificial Analysis" - }, - { - "value": "design-arena", - "label": "Design Arena" - }, - { - "value": "openrouter", - "label": "OpenRouter evals" - } - ] + "isMulti": false, + "isClearable": true, + "data": { + "source": "fixed", + "values": [ + { + "value": "artificial-analysis", + "label": "Artificial Analysis" + }, + { + "value": "design-arena", + "label": "Design Arena" + }, + { + "value": "openrouter", + "label": "OpenRouter evals" + } + ] + } } ], "metadata": [ diff --git a/plugins/OpenRouter/v1/dataStreams/rankings.json b/plugins/OpenRouter/v1/dataStreams/rankings.json index 7791d527..30269124 100644 --- a/plugins/OpenRouter/v1/dataStreams/rankings.json +++ b/plugins/OpenRouter/v1/dataStreams/rankings.json @@ -24,23 +24,23 @@ }, { "key": "period", - "value": "{{ (p => /^(category|language):/.test((typeof segment !== 'undefined' && segment ? segment : '')) && (!p || p === 'day') ? 'week' : p || null)(typeof period !== 'undefined' ? period : null) }}" + "value": "{{ typeof segment === 'string' && /^(category|language):/.test(segment) && (typeof period !== 'string' || !period || period === 'day') ? 'week' : typeof period === 'string' && period ? period : null }}" }, { "key": "category", - "value": "{{ (s => s.startsWith('category:') ? s.slice(9) : null)((typeof segment !== 'undefined' && segment ? segment : '')) }}" + "value": "{{ typeof segment === 'string' && segment.startsWith('category:') ? segment.slice(9) : null }}" }, { "key": "language_type", - "value": "{{ (s => s.startsWith('language:') ? s.slice(9) : null)((typeof segment !== 'undefined' && segment ? segment : '')) }}" + "value": "{{ typeof segment === 'string' && segment.startsWith('language:') ? segment.slice(9) : null }}" }, { "key": "modality", - "value": "{{ (s => s.startsWith('modality:') ? s.slice(9) : null)((typeof segment !== 'undefined' && segment ? segment : '')) }}" + "value": "{{ typeof segment === 'string' && segment.startsWith('modality:') ? segment.slice(9) : null }}" }, { "key": "context_bucket", - "value": "{{ (s => s.startsWith('context:') ? s.slice(8) : null)((typeof segment !== 'undefined' && segment ? segment : '')) }}" + "value": "{{ typeof segment === 'string' && segment.startsWith('context:') ? segment.slice(8) : null }}" } ], "headers": [], @@ -52,6 +52,7 @@ "type": "choiceChips", "name": "period", "label": "Period", + "defaultValue": "day", "options": [ { "value": "day", @@ -71,6 +72,7 @@ "type": "autocomplete", "name": "segment", "label": "Segment (optional)", + "isMulti": false, "isClearable": true, "help": "Shows weekly estimates for **Use case** and **Language** segments, so a **Day** period is shown weekly", "data": { diff --git a/plugins/OpenRouter/v1/dataStreams/sessionCost.json b/plugins/OpenRouter/v1/dataStreams/sessionCost.json index efe8a9ca..2a4a060d 100644 --- a/plugins/OpenRouter/v1/dataStreams/sessionCost.json +++ b/plugins/OpenRouter/v1/dataStreams/sessionCost.json @@ -113,6 +113,7 @@ "type": "autocomplete", "name": "harness", "label": "Harness", + "isMulti": false, "isClearable": true, "allowCustomValues": true, "help": "Select a coding-agent harness, or enter the slug of another harness OpenRouter publishes", @@ -139,27 +140,32 @@ } }, { - "type": "choiceChips", + "type": "autocomplete", "name": "turnRange", "label": "Session length", - "options": [ - { - "value": "1-turn", - "label": "1 turn" - }, - { - "value": "2-9-turns", - "label": "2\u20139 turns" - }, - { - "value": "10-49-turns", - "label": "10\u201349 turns" - }, - { - "value": "50-plus-turns", - "label": "50+ turns" - } - ] + "isMulti": false, + "isClearable": true, + "data": { + "source": "fixed", + "values": [ + { + "value": "1-turn", + "label": "1 turn" + }, + { + "value": "2-9-turns", + "label": "2\u20139 turns" + }, + { + "value": "10-49-turns", + "label": "10\u201349 turns" + }, + { + "value": "50-plus-turns", + "label": "50+ turns" + } + ] + } }, { "type": "objects",