diff --git a/leaderboard/serve.py b/leaderboard/serve.py new file mode 100644 index 0000000..91b905b --- /dev/null +++ b/leaderboard/serve.py @@ -0,0 +1,25 @@ +import http.server +import socketserver +import mimetypes + +# Force correct MIME types for JavaScript modules +mimetypes.init() +mimetypes.add_type('application/javascript', '.js') +mimetypes.add_type('application/javascript', '.mjs') +mimetypes.add_type('text/css', '.css') +mimetypes.add_type('image/svg+xml', '.svg') +mimetypes.add_type('application/json', '.json') + +PORT = 8000 + +class Handler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory='site', **kwargs) + + def guess_type(self, path): + mime, _ = mimetypes.guess_type(path) + return mime or 'application/octet-stream' + +with socketserver.TCPServer(("", PORT), Handler) as httpd: + print(f"Serving at http://localhost:{PORT}") + httpd.serve_forever() diff --git a/leaderboard/site/assets/css/components.css b/leaderboard/site/assets/css/components.css index eb57a6d..a19bc49 100644 --- a/leaderboard/site/assets/css/components.css +++ b/leaderboard/site/assets/css/components.css @@ -185,16 +185,14 @@ font-weight: 500; } -/* Hero — centered text stack, no rounded clip. Background lives on - body::before so the gradient bleeds into the page naturally. */ +/* Hero */ .hero { - padding: 3.25rem 1rem 2.75rem; + padding: 4rem 1.5rem 3rem; margin-bottom: 1.5rem; - text-align: center; } .hero h1 { - margin: 0 auto 0.55rem; - font-size: 3rem; + margin: 0 0 0.55rem; + font-size: 2.75rem; font-weight: 700; color: var(--fg-strong); letter-spacing: -0.025em; @@ -206,20 +204,90 @@ white-space: nowrap; } .hero .hero-sub { - margin: 0 auto 1.4rem; - font-size: 1.15rem; - color: var(--fg-strong); + margin: 0 0 0; + font-size: 1.05rem; + color: var(--fg-muted); font-weight: 400; letter-spacing: -0.005em; - line-height: 1.35; - white-space: nowrap; + line-height: 1.45; } -.hero .tagline { - color: var(--fg-muted); - margin: 0 auto 1.9rem; - font-size: 1rem; - line-height: 1.6; - max-width: 54ch; + +/* Hero: side-by-side text + comparison chart */ +.hero-main { + display: flex; align-items: center; justify-content: center; + gap: 2rem; margin: 0 auto 1.6rem; +} +.hero-text { flex: 0 0 auto; } +.hero-text h1 { max-width: none; } +.hero-text .hero-sub { max-width: none; white-space: nowrap; } + +/* Chart card */ +.hero-chart-card { + flex-shrink: 0; + background: var(--bg-elev); + border: 1px solid var(--border-soft); + border-radius: var(--r-md, 8px); + padding: 1rem 1.15rem 0.9rem; +} +.hero-chart-eyebrow { + display: block; font-size: 0.58rem; text-transform: uppercase; + letter-spacing: 0.1em; color: var(--fg-muted); margin-bottom: 0.5rem; + text-align: center; font-weight: 600; +} +.hero-chart-row { + display: flex; gap: 0.8rem; align-items: flex-end; +} +.hero-chart-panel { text-align: center; } +.hero-chart-panel-title { + display: block; font-size: 0.62rem; text-transform: uppercase; + letter-spacing: 0.06em; color: var(--fg-muted); margin-bottom: 0.3rem; + font-weight: 600; +} +.hero-chart-panel-note { + display: block; font-size: 0.7rem; margin-top: 0.3rem; + font-weight: 600; line-height: 1.3; +} +.hero-chart-panel-note.advantage { + color: #16a34a; +} +.hero-chart-panel-note.disadvantage { + color: #dc2626; +} +.winner-chip { + display: inline-block; + padding: 0.05rem 0.4rem; + background: color-mix(in srgb, var(--accent-2) 14%, transparent); + color: var(--accent-2); + border-radius: 3px; + font-weight: 600; + font-size: 0.68rem; + letter-spacing: 0.01em; +} +.hero-chart-caption { + font-size: 0.7rem; color: var(--fg-muted); margin-top: 0.65rem; + text-align: center; font-weight: 400; line-height: 1.5; max-width: 42ch; + margin-left: auto; margin-right: auto; +} + +@media (max-width: 840px) { + .hero-main { flex-direction: column; gap: 1.5rem; } + .hero-text { text-align: center; } + .hero-text .hero-sub { margin-left: auto; margin-right: auto; max-width: 44ch; white-space: normal; } +} +@media (max-width: 560px) { + .hero { padding: 2.5rem 1rem 2rem; } + .hero h1 { font-size: 2rem; } + .hero-chart-row { gap: 0.3rem; } + .hero-chart-panel-title { font-size: 0.56rem; } +} + +.hero-reversal-hint { + color: var(--accent); + margin: 0 auto 1.5rem; + font-size: 0.92rem; + font-weight: 500; + max-width: 50ch; + opacity: 0.9; } .hero-stats { diff --git a/leaderboard/site/assets/css/home.css b/leaderboard/site/assets/css/home.css index b91bed0..8a54239 100644 --- a/leaderboard/site/assets/css/home.css +++ b/leaderboard/site/assets/css/home.css @@ -186,6 +186,11 @@ .suite-card .lb-row-chip .lb-row-fw .fw-ver { font-size: 0.88em; } +.suite-card .lb-row-chip .lb-row-fw .fw-ops { + font-size: 0.82em; + color: var(--accent); + font-weight: 500; +} .suite-card .lb-row-sub { font-size: 0.68rem; line-height: 1.2; @@ -324,6 +329,11 @@ font-size: 0.85em; color: var(--fg-faint); } +.lb-row-chip .lb-row-fw .fw-ops { + font-size: 0.82em; + color: var(--accent); + font-weight: 500; +} .lb-row-name:hover { color: var(--accent-2); text-decoration: underline; diff --git a/leaderboard/site/assets/css/layout.css b/leaderboard/site/assets/css/layout.css index 5ddfb8e..a60429e 100644 --- a/leaderboard/site/assets/css/layout.css +++ b/leaderboard/site/assets/css/layout.css @@ -108,7 +108,7 @@ footer a:hover { color: var(--fg); border-bottom-color: var(--fg); text-decorati color: var(--fg-muted); font-size: 0.92rem; line-height: 1.45; - text-align: right; + text-align: left; flex: 0 1 auto; } diff --git a/leaderboard/site/assets/css/pages.css b/leaderboard/site/assets/css/pages.css index 0579a78..65e7500 100644 --- a/leaderboard/site/assets/css/pages.css +++ b/leaderboard/site/assets/css/pages.css @@ -2,8 +2,8 @@ .page-hero { text-align: center; - padding: 2rem 1rem 1.5rem; - max-width: 48rem; + padding: 2rem 2rem 1.5rem; + max-width: 72rem; margin: 0 auto; } .page-hero .eyebrow { display: block; margin-bottom: 0.5rem; } @@ -19,7 +19,7 @@ color: var(--fg-muted); font-size: 1rem; line-height: 1.6; - max-width: 42rem; + max-width: none; margin: 0 auto 1.25rem; } .hero-stats.compact { margin-bottom: 1rem; padding: 0.75rem 0; } diff --git a/leaderboard/site/assets/css/rankings.css b/leaderboard/site/assets/css/rankings.css index b0bc0b1..c984172 100644 --- a/leaderboard/site/assets/css/rankings.css +++ b/leaderboard/site/assets/css/rankings.css @@ -473,6 +473,11 @@ font-size: 0.85em; color: var(--fg-faint); } +.data-table td.col-chip .rk-chip-fw .fw-ops { + font-size: 0.82em; + color: var(--accent); + font-weight: 500; +} .data-table td.col-chip .rk-chip-meta { display: block; margin-top: 0.1rem; @@ -653,6 +658,22 @@ color: var(--fg-muted); letter-spacing: 0.005em; } +.cmp-basket-chip-select { + font-size: 0.7rem; + font-weight: 500; + color: var(--fg-muted); + background: var(--bg-elev); + border: 1px solid var(--border-soft); + border-radius: var(--r-sm, 4px); + padding: 0.1rem 0.25rem; + cursor: pointer; + max-width: 20ch; + font-family: inherit; +} +.cmp-basket-chip-select:focus-visible { + outline: 2px solid var(--accent-2); + outline-offset: 1px; +} .cmp-basket-chip .vendor-dot { display: inline-block; width: 8px; diff --git a/leaderboard/site/assets/css/suites.css b/leaderboard/site/assets/css/suites.css index 6a2ec5c..026c9b6 100644 --- a/leaderboard/site/assets/css/suites.css +++ b/leaderboard/site/assets/css/suites.css @@ -107,6 +107,12 @@ line-height: 1.7; margin: 0 0 1rem; } +.why-prose .why-lead { + color: var(--accent); + font-size: 1.08rem; + line-height: 1.65; + margin-bottom: 1.25rem; +} .why-prose p:last-child { margin-bottom: 0; } .why-prose strong { color: var(--fg-strong); font-weight: 600; } .why-prose em { color: var(--fg-strong); font-style: italic; } diff --git a/leaderboard/site/assets/js/community-nav.js b/leaderboard/site/assets/js/community-nav.js index 2b8f9c3..8254412 100644 --- a/leaderboard/site/assets/js/community-nav.js +++ b/leaderboard/site/assets/js/community-nav.js @@ -1,18 +1,19 @@ // community-nav.js — shared navigation for community sub-pages. import { esc } from "./utils.js"; +const _i = (k, r) => (window._i ? window._i(k, r) : k); -const TABS = [ - { id: "contributors", href: "#/contributors", label: "Contributors" }, - { id: "wanted", href: "#/wanted", label: "Wanted hardware" }, - { id: "reproduce", href: "#/reproduce", label: "Reproduction quests" }, -]; +function getTabs() { return [ + { id: "contributors", href: "#/contributors", label: _i('nav.contributors') }, + { id: "wanted", href: "#/wanted", label: _i('nav.wanted') }, + { id: "reproduce", href: "#/reproduce", label: _i('nav.reproduce') }, +]; } /** Underline tab strip — one row, no subtitles. */ export function communityTabs(activeId) { return ` @@ -79,8 +80,8 @@ export function wireTableFilters(root, opts) { } if (countEl) { countEl.textContent = visible === rows.length - ? `${visible} entries` - : `${visible} of ${rows.length} entries`; + ? `${visible} ${_i('community.entries')}` + : `${visible} ${_i('community.of')} ${rows.length} ${_i('community.entries')}`; } } diff --git a/leaderboard/site/assets/js/data.js b/leaderboard/site/assets/js/data.js index 83238a7..9a9d5ff 100644 --- a/leaderboard/site/assets/js/data.js +++ b/leaderboard/site/assets/js/data.js @@ -122,6 +122,29 @@ const FALLBACK_PALETTE = [ "#a78bfa", "#2dd4bf", "#fbbf24", "#fb7185", "#22d3ee", ]; +// Return a compact operator/kernel label from the impl dict. +// Extracts backend and notable kernels so users can distinguish +// different implementations of the same framework at a glance. +export function implOpsLabel(row) { + const imp = row && row.impl; + if (!imp || typeof imp !== "object") return ""; + const desc = (imp.description || "") + " " + (imp.notes || ""); + const parts = []; + // backend + if (/JAX\/XLA|tpu-inference/i.test(desc)) parts.push("JAX/XLA"); + else if (/CANN|torch_npu|Ascend/i.test(desc)) parts.push("CANN"); + else if (/Metal\b|Metal\./i.test(desc)) parts.push("Metal"); + else if (/MUSA/i.test(desc)) parts.push("MUSA"); + else parts.push("CUDA"); + // notable kernels / constraints + if (/FLASH_ATTN_V100/i.test(desc)) parts.push("FlashAttn-V100"); + if (/AWQ\s*SM70|SM70\s*AWQ/i.test(desc)) parts.push("AWQ SM70"); + if (/turboquant/i.test(desc)) parts.push("turboquant"); + if (/BF16\s*only|FP8.*not\s*supported/i.test(desc)) parts.push("BF16 only"); + if (/Standard vLLM|no custom patches/i.test(desc)) parts.push("std"); + return parts.join(" · "); +} + export function vendorColor(name) { if (!name) return "#888780"; if (VENDOR_COLORS[name]) return VENDOR_COLORS[name]; diff --git a/leaderboard/site/assets/js/i18n.js b/leaderboard/site/assets/js/i18n.js new file mode 100644 index 0000000..9651965 --- /dev/null +++ b/leaderboard/site/assets/js/i18n.js @@ -0,0 +1,857 @@ +// i18n.js — Translation data loaded into the _i18n bootstrap +window._i18n.set({ + en: { + 'nav.home': 'Home', 'nav.results': 'Results', 'nav.compare': 'Compare', 'nav.suites': 'Suites', + 'nav.community': 'Community', 'nav.submit': 'Submit', 'nav.lang': '中文', + 'nav.contributors': 'Contributors', 'nav.wanted': 'Wanted hardware', 'nav.reproduce': 'Reproduction quests', + 'footer.tagline': 'AccelMark · Open-source LLM inference benchmarking.', + 'footer.cite': 'Cite dataset', 'footer.api': 'API manifest', + 'footer.source': 'Source code', 'footer.discuss': 'Discussions', + 'hero.title1': 'Run your accelerator.', + 'hero.title2': 'Publish reproducible LLM inference benchmarks.', + 'hero.subtitle': 'No single fastest chip. Rankings reverse across short-chat, long-context, MoE, and edge — only the best recipe per workload.', + 'hero.tagline': 'Every row links to result.json, env_info.json, runner hash, accuracy receipt, and reproduction instructions — open evidence, not just a number on a chart.', + 'hero.reversal': 'Rankings reverse: what wins on short-prompt chat may lose on long-context prefill.', + 'hero.kpi.benchmarks': 'benchmarks', 'hero.kpi.gpus': 'platforms', 'hero.kpi.vendors': 'vendors', + 'hero.kpi.workloads': 'workloads', 'hero.kpi.verified': 'verified', 'hero.kpi.thisWeek': 'this week', + 'hero.cta.submit': 'Submit a result →', 'hero.cta.compare': 'Compare chips', 'hero.cta.cite': 'Cite the dataset', + 'hero.chartEyebrow': 'Rankings reverse', + 'hero.chartCaption': 'Same vLLM framework. 5090 wins on bandwidth-bound short chat. A100 wins on compute-bound long context. The ranking flips.', + 'why.eyebrow': 'Why submit', 'why.title': 'Get a shareable, citable public record', + 'why.card1.title': 'Reproducible evidence', 'why.card1.desc': 'Your run ships with environment fingerprint, runner source hash, and validation receipts — others can rerun and verify, not just trust a screenshot.', + 'why.card2.title': 'First results on new hardware', 'why.card2.desc': 'Missing your platform? See wanted hardware.
Have matching hardware? Browse reproduction quests.', + 'why.card3.title': 'Citable submissions', 'why.card3.desc': 'Papers and reports can cite project-level, snapshot-level, or per-run BibTeX — with deep links back to your submission artifacts.', + 'dist.eyebrow': '01 · Explore', 'dist.title': 'The recipe landscape', + 'dist.subtitle': 'Each dot is a serving recipe — framework, precision, and hardware together. Wider spread within a suite means more room for tuning.', + 'dist.filter.suite': 'Suite', 'dist.filter.vendor': 'Vendor', 'dist.filter.framework': 'Framework', + 'dist.filter.precision': 'Precision', 'dist.filter.chip': 'Chip', 'dist.filter.reset': '↺ Reset', 'dist.all': 'All', + 'dist.metric.label': 'Metric', 'dist.metric.offline': 'Offline Throughput', 'dist.metric.online': 'Online Max QPS', + 'dist.metric.sustained': 'Sustained Throughput', 'dist.metric.speculative': 'Speculative Throughput', + 'dist.tab.byframework': 'By Framework', 'dist.normalize': 'Normalize', 'dist.moreviews': 'More views', + 'dist.info.all': 'Beeswarm — Grouped by suite; each dot is a recipe. Larger dots = best per vendor.
By Framework — Same layout, grouped by serving framework.
Scatter — Throughput × QPS. Top-right = both strong.
Density — Overlap shading. Darker = more recipes.
Heatmap — Suite × Chip matrix. Color = throughput.
By Suite — One mini-chart per suite.
Normalize — Column views only: Y-axis becomes % of column best (0–100%).', + 'dist.info.smallchart': 'Each suite in its own chart', + 'dist.info.showing': 'Showing {n} recipes across {s} suites · {m}', + 'dist.caption.beeswarm': 'Each point is one serving recipe; larger markers denote best-in-class per vendor. Y-axis log-scaled.', + 'dist.caption.beeswarm_norm': 'Each point is one serving recipe; larger markers denote best-in-class per vendor. Y-axis shows % of column best (linear).', + 'dist.caption.scatter': 'Throughput (x) vs online QPS (y); each point is one serving recipe. Both axes log-scaled.', + 'dist.caption.density': 'Kernel-density-style overlap of recipe throughput; darker regions indicate more submissions.', + 'dist.caption.heatmap': 'Best throughput per chip × suite cell; color intensity encodes the selected metric.', + 'dist.caption.small': 'Per-suite mini scatter plots (throughput × QPS); each point is one serving recipe.', + 'workloads.eyebrow': '02 · Workloads', 'workloads.title': 'Browse by benchmark recipe', + 'workloads.subtitle': 'Each serving recipe tests a fixed model under a specific protocol — from single-GPU throughput to long-context serving.', + 'workloads.viewfull': 'View all results →', 'workloads.awaiting': 'Awaiting first submission.', + 'coverage.eyebrow': '03 · Coverage', 'coverage.title': 'Platform coverage', + 'coverage.subtitle': 'Tile size reflects submission count. Colour indicates vendor family.', + 'recent.eyebrow': '04 · Latest activity', 'recent.title': 'Recent submissions', 'recent.seeall': 'See all →', + 'community.eyebrow': '05 · Community', 'community.title': 'Contribution index', 'community.all': 'All contributors →', + 'community.subtitle': 'Ranked by verified runs and reproducible evidence — each submission links to full artifacts.', + 'community.rank': 'Rank', 'community.contributor': 'Contributor', 'community.runs': 'Runs', 'community.verified': 'Verified', 'community.score': 'Score', 'community.empty': 'No contributors yet.', + 'contribute.eyebrow': '06 · Contribute', 'contribute.title': 'Ready to publish your benchmark?', + 'contribute.body': 'If you already have the hardware and a supported serving stack, you can go from zero to a merged PR in about ten minutes.', + 'contribute.step1': 'Open the submit wizard — pick workload suite, platform, and framework; it prints the exact command and output folder layout.', + 'contribute.step2': 'Run the benchmark — one command produces result.json, env_info.json, and accuracy receipts.', + 'contribute.step3': 'Open a pull request — add that folder; CI checks artifacts. After merge you get a permanent link and citation exports.', + 'contribute.foot': 'First time with the harness, accuracy rules, or hardware setup? ', + 'contribute.guide': 'Read the full contributor guide ↗', + 'contribute.cta1': 'Open submit wizard →', 'contribute.cta2': 'Wanted hardware', 'contribute.cta3': 'Reproduction quests', + 'rankings.eyebrow': 'Results · Suite {letter}', + 'rankings.subtitle': 'Compare configurations across frameworks and hardware.', + 'rankings.table.recipe': 'Recipe', 'rankings.table.vendor': 'Vendor', 'rankings.table.precision': 'Precision', + 'rankings.table.framework': 'Framework', 'rankings.table.date': 'Date', 'rankings.table.tier': 'Tier', + 'rankings.table.throughput': 'Best Throughput', 'rankings.table.headroom': 'Headroom', + 'rankings.of': '', 'rankings.results': 'results', 'rankings.sorted': 'Sorted by', + 'rankings.clear': 'Clear filters', 'rankings.clearFilters': 'Clear all filters', 'rankings.showall': 'Show all results', 'rankings.viewchip': 'View chip overview', + 'rankings.basket.run': 'run', 'rankings.basket.runs': 'runs', 'rankings.basket.msg': 'in your compare basket', + 'rankings.basket.compare': 'Open compare →', 'rankings.basket.clear': 'Clear basket', + 'rankings.empty': 'No submissions yet.', 'rankings.browse': 'Browse rankings →', + 'suites.hero.title': 'Workload Suites', 'suites.hero.sub': 'Rankings reverse across regimes. Each suite isolates one bottleneck region—memory-bound, compute-bound, latency-sensitive—so you see which recipe wins where.', + 'suites.hero.cta1': 'Browse results →', 'suites.hero.cta2': 'Suite spec on GitHub', + 'suites.s1.eyebrow': '01 · Methodology', 'suites.s1.title': 'Why per-suite, not a single score?', + 'suites.s1.lead': 'Rankings reverse across regimes. A chip that dominates short-prompt throughput may lose on long-context prefill. Each suite isolates one bottleneck—memory-bound, compute-bound, or latency-sensitive—so you see which recipe wins where.', + 'suites.s1.body': 'AI inference workloads span a wide range of arithmetic intensity. The roofline model makes the consequence concrete: a chip\'s effective performance is set by whichever of memory bandwidth or compute is binding for the workload. Because different workloads occupy different regions of that spectrum, hardware rankings are not preserved across them.', + 'suites.s2.eyebrow': '02 · Scenarios', 'suites.s2.title': 'Seven protocols, one suite at a time', + 'suites.s3.eyebrow': '03 · Specifications', 'suites.s3.title': 'Each suite, in detail', + 'suites.s4.eyebrow': '04 · Datasets', 'suites.s4.title': 'Three immutable prompt sets', + 'compare.eyebrow': 'Compare', 'compare.title': 'Side-by-Side Comparison', + 'compare.sub1': 'Pick platforms below to start a head-to-head across every metric. You can also tick runs from', + 'compare.sub2': 'any results page', 'compare.sub3': 'to compare specific framework and precision configurations.', + 'compare.quickadd': 'Quick-add by platform', + 'compare.quickaddHint': 'Each click adds that platform\'s most recent run. Choose any two or more.', + 'compare.empty': 'No runs in your compare basket yet.', + 'submit.eyebrow': 'Contribute', 'submit.hero': 'Submit your benchmark', + 'contributors.title': 'Contributors', + 'contributors.count': 'contributors', + 'contributors.ranking': 'Ranking', + 'contributors.scoreDesc': 'Score = runs + verified×3 + platforms×4 + first-result×12 + runner×5', + 'contributors.rank': 'Rank', 'contributors.contributor': 'Contributor', 'contributors.runs': 'Runs', + 'contributors.verified': 'Verified', 'contributors.platforms': 'Platforms', 'contributors.attribs': 'Attributions', + 'contributors.score': 'Score', 'contributors.empty': 'No contributors yet', + 'contributors.submitFirst': 'submit the first result', + 'contributors.attribDefs': 'Attribution definitions', + 'chipDetail.notFound': 'No chip found for', + 'chipDetail.backToHome': 'Back to home', 'chipDetail.browseResults': 'Browse results', + 'chipDetail.copyLink': 'Copy link', 'chipDetail.results': 'results', + 'chipDetail.bestPerSuiteTitle': 'Best result per suite', + 'chipDetail.fingerprintTitle': 'How this chip sits across the spectrum', + 'chipDetail.scalingTitle': 'Does going wide actually pay off?', + 'chipDetail.peersTitle': 'Compare with similar chips', + 'chipDetail.thSuite': 'Suite', 'chipDetail.thChips': 'Chips', + 'chipDetail.thFramework': 'Framework', 'chipDetail.thPrecision': 'Precision', + 'chipDetail.thDate': 'Date', 'chipDetail.thSubmitter': 'Submitter', + 'chipDetail.thTier': 'Tier', + 'chipDetail.thPrimaryMetric': 'Primary Metric', + 'chipDetail.suiteUnit': 'suite', 'chipDetail.suiteUnitPlural': 'suites', + 'chipDetail.runUnit': 'run', 'chipDetail.runUnitPlural': 'runs', + 'chipDetail.frameworkUnit': 'framework', 'chipDetail.frameworkUnitPlural': 'frameworks', + 'chipDetail.precisionUnit': 'precision', 'chipDetail.precisionUnitPlural': 'precisions', + 'chipDetail.chipCountVariants': 'chip-count variants', + 'chipDetail.deployedAt': 'Deployed at', + 'chipDetail.compareConfigs': 'Compare configs for this chip', + 'chipDetail.browseVendor': 'Browse all', + 'chipDetail.copyLinkTitle': 'Copy link to this chip overview', + 'chipDetail.bestPerSuiteEyebrow': 'Best per suite', + 'chipDetail.bestPerSuiteSub': 'Top result submitted for each benchmark suite.', + 'chipDetail.bestScoreCountTitle': 'Best score count by suite', + 'chipDetail.ofBest': 'of best', + 'chipDetail.globalBest': 'Global best', + 'chipDetail.rankedInSuite': 'ranked in suite', + 'chipDetail.noSubmission': 'No submission', + 'chipDetail.noSubmissionAt': 'No submission at this chip count.', + 'chipDetail.notSubmitted': 'Not submitted', + 'chipDetail.openRun': 'Open run', + 'chipDetail.openRunDetails': 'Open run details', + 'chipDetail.cmdClickAllInSuite': 'Cmd/Ctrl + click to browse all results in this suite.', + 'chipDetail.everySubEyebrow': 'submissions', + 'chipDetail.runsOnFile': 'runs on file', + 'chipDetail.runsOnFileSub': 'Every submitted run across all suites, sorted by date.', + 'chipDetail.downloadRadarTitle': 'Download radar chart as PNG', + 'chipDetail.downloadScalingTitle': 'Download scaling chart as PNG', + 'chipDetail.png': 'PNG', + 'chipDetail.flashCopied': 'Copied!', + 'chipDetail.flashCopyFailed': 'Copy failed', + 'chipDetail.flashFailed': 'Download failed', + 'chipDetail.flashSaved': 'Saved!', + 'chipDetail.fingerprintEyebrow': 'Fingerprint', + 'chipDetail.fingerprintSub1': 'Each axis is a benchmark suite. Further from centre = higher throughput. A chip that dominates one workload may trail on another.', + 'chipDetail.fingerprintSubMissing': 'Suites without a submission collapse to the centre ({missing}).', + 'chipDetail.radarAriaLabel': 'Radar chart of {chip} performance across all suites', + 'chipDetail.fpLegendAria': 'Suite legend for the fingerprint radar', + 'chipDetail.scalingEyebrow': 'Scaling', + 'chipDetail.scalingSub': 'Throughput per chip as chip count increases. Ideally linear; real workloads see diminishing returns.', + 'chipDetail.scalingChartAria': 'Per-chip scaling chart', + 'chipDetail.sclCellTitle': 'Scaling breakdown by suite and chip count', + 'chipDetail.sclBreakdownAria': 'Scaling efficiency table', + 'chipDetail.sclLegendAria': 'Metric legend for scaling chart', + 'chipDetail.peersEyebrow': 'Peers', + 'chipDetail.peersSub': 'Chips with similar specs — same vendor, memory, or generation.', + 'chipDetail.cardTitle': 'Suites with at least one submission', + 'chipDetail.stale': 'This chip may have been renamed or removed from the dataset.', + 'submit.heroSub': 'From hardware detection to a published, citable result.', + 'submit.flow_detect': 'Detect', 'submit.flow_run': 'Run', + 'submit.flow_validate': 'Validate', 'submit.flow_preview': 'Preview', + 'submit.flow_submit': 'Submit', 'submit.flow_published': 'Published', + 'submit.step_hardware': 'Hardware', 'submit.step_suite': 'Suite', + 'submit.step_run': 'Run', 'submit.step_submit': 'Submit', + 'submit.chooseHardware': 'Choose your hardware', + 'submit.searchCatalog': 'Search catalog', 'submit.vendor': 'Vendor', + 'submit.gpuAccelerator': 'GPU / accelerator', 'submit.chipCount': 'Chip count', + 'submit.pickSuite': 'Pick a suite', 'submit.runValidate': 'Run & validate', + 'submit.install': 'Install', 'submit.copy': 'Copy', + 'submit.benchmark': 'Benchmark', 'submit.validateLocally': 'Validate locally', + 'submit.submitForPublication': 'Submit for publication', + 'submit.submitForPublicationSub': 'Every merged result links to result.json, env_info.json, and reproduction instructions.', + 'submit.prRecommendedSub': 'Fork → run benchmark → open PR.', + 'submit.uploadIssueSub': 'Attach result.json + env_info.json.', + 'submit.shareDiscussionsSub': 'Show and tell — hardware tips, optimization notes.', + 'submit.searchPlaceholder': 'e.g. H100, MI350, Ascend 910C, RTX 5090', + 'submit.chooseHardwareSub': 'Full catalog - datacenter, workstation, consumer.', + 'submit.pickSuiteHint': 'New contributors: start with Suite A or Suite F (~10 min).', + 'submit.firstBenchmark': 'First benchmark in ~11 minutes on datacenter GPUs.', + 'submit.customName': 'Or custom name', 'submit.customPlaceholder': 'If not listed above', + 'submit.artifactsPerRun': 'Artifacts per run:', + 'submit.done': 'Done', 'submit.estimatedRuntime': 'Estimated runtime', + 'submit.suggestedPrefix': 'suggested', 'submit.recommended': 'Recommended', + 'submit.suiteHint.A': 'Best first benchmark · single-chip inference', + 'submit.suiteHint.B': 'Large-model multi-chip', 'submit.suiteHint.C': 'Quantization efficiency', + 'submit.suiteHint.D': 'Long-context inference', 'submit.suiteHint.E': 'Multi-chip scaling', + 'submit.suiteHint.F': 'Edge / consumer GPU · offline + online', 'submit.suiteHint.G': 'MoE multi-chip', + 'submit.flashCopied': 'Copied!', 'submit.flashFailed': 'Failed', + 'submit.prRecommended': 'Pull request (recommended)', + 'submit.uploadIssue': 'Upload via GitHub Issue', + 'submit.shareDiscussions': 'Share in Discussions', + 'submit.back': 'Back', 'submit.next': 'Next', + 'compare.stalePrefix': 'This comparison link refers to', + 'compare.aRun': 'a run', 'compare.runs': 'runs', + 'compare.staleMid': 'that', + 'compare.staleSuffix': 'no longer exist in the dataset.', + 'compare.comparing': 'Comparing', 'compare.stale': 'stale', 'compare.run': 'run', + 'compare.clearStartOver': 'Clear & start over', + 'compare.quickAddPlatform': 'Quick-add by platform', + 'compare.quickAddPlatformHint': 'Click any chip to add its best run to the compare basket.', + 'compare.clearAll': 'Clear all', 'compare.suite': 'Suite', + 'compare.noSubmissions': 'The platforms you picked have no submissions on file.', + 'compare.metric': 'Metric', 'compare.headToHeadCharts': 'Head-to-head charts', + 'compare.copyShareLink': 'Copy share link', + 'compare.copyShareLinkTitle': 'Copy a shareable link for this comparison', + 'compare.downloadChartTitle': 'Download chart as PNG', + 'compare.png': 'PNG', + 'compare.flashCopied': 'Copied!', + 'compare.flashCopyFailed': 'Copy failed', + 'compare.flashFailed': 'Download failed', + 'compare.flashSaved': 'Saved!', + 'compare.across': 'across', + 'compare.addTo': 'add to', + 'compare.chipCountVariants': 'chip-count variants', + 'compare.clickTo': 'Click to', + 'compare.compareBasketCtx': 'compare basket; Cmd / Ctrl-click to open chip overview.', + 'compare.removeFrom': 'remove from', + 'compare.submissionUnit': 'submission', + 'compare.submissionUnitPlural': 'submissions', + 'compare.suiteUnit': 'suite', + 'compare.suiteUnitPlural': 'suites', + 'compare.higherIsBetter': 'Higher is better', + 'compare.lowerIsBetter': 'Lower is better', + 'compare.chartsHintOf': 'of', 'compare.chartsHintChips': 'chips', + 'compare.chartSingleChipOffline': 'Single-Chip Offline Throughput', + 'compare.chartTokByConcurrency': 'Tokens/s by Concurrency', + 'compare.noSuiteData': 'No suite data available for the selected chips.', + 'compare.noSuiteData1': 'The selected chips have no data in ', + 'compare.noSuiteData2': '.', + 'compare.try': 'Try ', + 'compare.instead': ' instead.', + 'community.entries': 'entries', + 'community.of': 'of', + 'compare.chartsHintAll': 'All selected chips have data for this suite.', + 'compare.chartFailed': 'Chart failed to render.', + 'compare.chartEdgeOffline': 'Edge Offline Throughput', + 'compare.chartMoeOffline': 'MoE Offline Throughput', + 'compare.chartMultiChipTotal': 'Multi-Chip Total Throughput', + 'compare.chartPeakOffline': 'Peak Offline Throughput', + 'compare.chartTokensPerSec': 'tokens/sec', + 'compare.chartConcurrency': 'Concurrency', + 'compare.chartPerChipThroughput': 'Per-Chip Throughput', + 'compare.chartPerChipSub': 'Total throughput ÷ chip count; higher = better single-chip efficiency.', + 'compare.chartTokPerSecPerChip': 'tokens/sec per chip', + 'compare.chartQuantThroughput': 'Throughput by Precision', + 'compare.chartQuantSub': 'Compares BF16 vs quantized formats (FP8, INT8, INT4).', + 'compare.chartPrecision': 'Precision', + 'compare.chartTtftP50': 'TTFT p50', 'compare.chartTtftP90': 'TTFT p90', 'compare.chartTtftP99': 'TTFT p99', + 'compare.chartTpotP50': 'TPOT p50', 'compare.chartTpotP90': 'TPOT p90', 'compare.chartTpotP99': 'TPOT p99', + 'compare.chartLongContextLatency': 'Long-Context Latency', + 'compare.chartLongContextSub': 'TTFT / TPOT percentiles at 28K tokens.', + 'compare.chartLatency': 'Latency', + 'compare.chartThroughputByCount': 'Throughput by Chip Count', + 'compare.chartThroughputByCountSub': 'How total tokens/sec scales with more chips.', + 'compare.chartScalingEfficiency': 'Scaling Efficiency', + 'compare.chartScalingEfficiencySub': '2× efficiency = (2-chip / 1-chip × 2) × 100.', + 'compare.chartLinearIdeal': 'Linear ideal (100%)', + 'compare.chartGood80': 'Good (80%)', + 'compare.chartEfficiencyPct': 'Efficiency %', + 'wanted.title': 'Wanted hardware', + 'wanted.desc1': 'Accelerators in our catalog not yet covered on the leaderboard', + 'wanted.open': 'open', 'wanted.covered': 'covered', + 'wanted.desc2': 'The first published result for a platform receives a First result attribution on the', + 'wanted.contribIndex': 'contributor index', + 'wanted.filterVendor': 'Vendor', 'wanted.filterTier': 'Tier', 'wanted.filterSearch': 'Search', + 'wanted.searchPlaceholder': 'Platform name', + 'wanted.thPlatform': 'Platform', 'wanted.thVendor': 'Vendor', 'wanted.thTier': 'Tier', + 'wanted.thMemory': 'Memory', 'wanted.thSuites': 'Recommended suites', + 'wanted.submit': 'Submit', 'wanted.allCovered': 'All catalog platforms are covered.', + 'wanted.allVendors': 'All vendors', 'wanted.allTiers': 'All tiers', + 'wanted.datacenter': 'Datacenter', 'wanted.cloud': 'Cloud', + 'wanted.workstation': 'Workstation', 'wanted.consumer': 'Consumer / edge', 'wanted.edge': 'Edge', + 'reproduce.title': 'Reproduction quests', + 'reproduce.desc': 'Independent reruns that confirm a community submission within 5% earn verifier credit for both parties. Reference the original run_id in your pull request.', + 'reproduce.whyTitle': 'Why these quests?', 'reproduce.verified': 'Verified', + 'reproduce.verifiedTitle': 'Cross-verified coverage', 'reproduce.awaitingTitle': 'Awaiting verification', + 'reproduce.open': 'Open', 'reproduce.procedure': 'Procedure', + 'reproduce.thOriginal': 'Original (community)', 'reproduce.thVerified': 'Verified rerun', + 'reproduce.thSuite': 'Suite', 'reproduce.thConfirmed': 'Confirmed', + 'reproduce.btnOriginal': 'Original', 'reproduce.btnVerified': 'Verified', + 'reproduce.btnDetails': 'Details', 'reproduce.btnRequest': 'Request', + 'reproduce.quest': 'quest', 'reproduce.quests': 'quests', + 'reproduce.pair': 'pair', 'reproduce.pairs': 'pairs', + 'reproduce.record': 'record', 'reproduce.records': 'records', + 'reproduce.filterVendor': 'Vendor', + 'reproduce.filterSuite': 'Suite', + 'reproduce.filterSearch': 'Search', + 'reproduce.searchPlaceholder': 'Platform, recipe, submitter', + 'reproduce.allVendors': 'All vendors', + 'reproduce.allSuites': 'All suites', + 'reproduce.thPlatform': 'Platform', + 'reproduce.thRecipe': 'Recipe', + 'reproduce.thSubmitter': 'Submitter', + 'reproduce.thDate': 'Date', + 'reproduce.thTier': 'Tier', + 'reproduce.empty': 'No open reproduction quests.', + 'reproduce.statOpen': '{n} open {w}', + 'reproduce.statLinked': '{n} linked verification {w}', + 'reproduce.statPairs': '{np} platform–suite {pw} awaiting any verified run · {nv} pairs with verified coverage', + 'reproduce.emptyVerifiedNote': 'The verifying run stores meta.reproduces_run_id, and this table fills automatically after the next generate.py refresh.', + 'reproduce.countLine': 'community runs on platforms without verified coverage yet.', + 'reproduce.emptyVerified': 'No linked verifications in this snapshot yet.', + 'reproduce.criteriaLead': 'The open list is automatic — nothing is hand-picked. A row appears when both conditions hold:', + 'reproduce.criteriaRule1': 'The submission is community tier (published, not yet independently verified).', + 'reproduce.criteriaRule2': 'No verified result exists for the same hardware platform + benchmark suite.', + 'reproduce.criteriaNote': 'Cross-verified coverage is separate: it only lists runs whose meta.reproduces_run_id explicitly cites the original community run_id.', + 'reproduce.countLine': 'community runs on platforms without verified coverage yet.', + 'reproduce.procedure1': 'Select a quest above and note its run_id.', + 'reproduce.procedure2': 'Rerun the same suite on matching hardware with --reproduces-run-id.', + 'reproduce.procedure3': 'Open a pull request; after merge, the pair appears below.', + 'reproduce.procedure4': 'Both submitters receive credit on the contributor index.', + 'badge.first-result.label': 'First result', 'badge.pioneer.label': 'Pioneer', + 'badge.verified.label': 'Verified', 'badge.verifier.label': 'Verifier', + 'badge.prolific.label': 'Prolific', 'badge.multi-chip.label': 'Multi-platform', + 'badge.multi-vendor.label': 'Cross-vendor', 'badge.runner.label': 'Runner author', + 'badge.first-result.desc': 'First published benchmark for a hardware platform', + 'badge.pioneer.desc': 'Among the first three contributors on the leaderboard', + 'badge.verified.desc': 'At least one verified-tier result', + 'badge.verifier.desc': 'Independent reproduction published as verified', + 'badge.prolific.desc': 'Ten or more published benchmark runs', + 'badge.multi-chip.desc': 'Results on three or more distinct hardware platforms', + 'badge.multi-vendor.desc': 'Results across two or more vendors', + 'badge.runner.desc': 'Credited on a runner implementation', + 'suites.readMore': 'Read the full argument', + 'suites.showLess': 'Show less', 'suites.concreteFinding': 'Concrete finding', + 'suites.currentLeaders': 'Current leaders', 'suites.openResults': 'Open results →', + 'suites.viewJson': 'View suite.json', 'suites.coverage': 'Coverage', + 'suites.protocols': 'Protocols', + 'suites.extra': 'extra', + 'suites.inversion.A.eyebrow': 'Offline vs Online', + 'suites.inversion.A.title': 'The throughput winner loses the SLA tier', + 'suites.inversion.A.body': 'Suite A: H200 leads offline at 5,731 tokens/sec, but caps at 10 queries/sec once the 500 ms p99 TTFT SLA is enforced. A100 and A800 sustain 25 queries/sec on the same hardware tier. Offline and online are not the same race.', + 'suites.inversion.G.eyebrow': 'Dense vs MoE', + 'suites.inversion.G.title': 'H20-3e: minus 5% on dense, plus 17% on MoE', + 'suites.inversion.G.body': 'On dense Suite A, H20-3e trails A100-40G by ~5%. On Suite G sparse Mixtral routing the same chip leads A100-40G ×8 by 17%. Sparse activation holds arithmetic intensity below dense 8B, so bandwidth (not FLOPS) sets the ceiling.', + 'suites.inversion.E.eyebrow': 'Multi-chip Amdahl', + 'suites.inversion.E.title': 'Newest flagship has the worst 2× scaling', + 'suites.inversion.E.body': 'RTX 4090 D tops the Suite E 2× efficiency table precisely because its per-die throughput is low; communication occupies a small share of wall time. H200 per-chip throughput outgrew NVLink 4.0 bandwidth, so its 2× scaling sits at the bottom of NVIDIA range.', + 'suites.scenario.accuracy.desc': 'MMLU subset score against the suite baseline model. Runs before any throughput scenario; a chip that drops accuracy beyond the threshold has every other number on this suite invalidated.', + 'suites.scenario.offline.desc': 'All requests submitted at once, no SLA, no concurrency cap. The pure capability number that establishes the chip ceiling on this workload.', + 'suites.scenario.online.desc': 'Sweeps offered load under Poisson arrivals and reports the highest queries/sec that still meets the 500 ms p99 TTFT SLA. The number production traffic actually has to honour.', + 'suites.scenario.interactive.desc': 'One request in-flight at a time, no concurrency. The chat-window UX baseline; minimal queueing, dominated by decode latency and software overhead.', + 'suites.scenario.sustained.desc': 'Fixed-concurrency load held for 15 to 30 minutes. Reports the throttle ratio between the first and last 60 s windows so thermal throttling and memory fragmentation surface.', + 'suites.scenario.speculative.desc': 'Offline workload with a 1B draft model loaded alongside the target. Reports speculative decoding acceptance rate and end-to-end speedup; tells you whether spec-decode is worth the VRAM cost.', + 'suites.scenario.burst.desc': 'Alternates 5× steady arrival rate (short windows) with steady traffic and reports TTFT p99 during the burst. Stresses the KV cache, admission control, and warm-up paths.', + 'suites.dataset.sharegpt_standard_v1.notes': 'Curated to match production LLM-API traffic; token-length p99 ≈ 2,100.', + 'suites.dataset.sharegpt_longctx_v1.notes': 'Multi-turn dialogues concatenated to push prefill past the roofline knee.', + 'suites.dataset.sharegpt_edge_v1.notes': 'Short single-turn prompts; keeps the edge suite bandwidth-isolated.', + 'suites.s5.eyebrow': '05 · Extend', 'suites.s5.title': 'Propose a new suite', + 'suites.s5.body': 'Have a workload regime AccelMark does not cover yet: long-context serving, speculative decoding economics, a domain-specific fine-tune? Open a discussion with a one-page sketch of the bottleneck region and reference SLAs. The contribution flow is the same as a new result.', + 'suites.s5.cta': 'Propose a suite →', + 'suites.noSubmissions': 'No qualifying submissions yet', + 'cite.title': 'Cite the dataset', 'cite.copyBibtex': 'Copy BibTeX', + 'cite.copyMarkdown': 'Copy Markdown', 'cite.apiTitle': 'Machine-readable API', + 'cite.lead': 'Three levels of citation — framework, versioned snapshot, and individual reproducible runs. All community results are CC BY 4.0.', + 'cite.singleResultDesc': 'Open any result on the Results page → use Copy link, Copy Markdown, or Copy BibTeX in the detail panel. Each run links to result.json, runner hash, and reproduction script.', + 'cite.browseResults': 'Browse results →', + 'cite.apiFuture': 'Future: /api/runs.json, /api/runs/<run_id>.json, versioned snapshots, Zenodo DOI per release.', + 'cite.projectDesc': 'Cite the AccelMark benchmarking framework and methodology.', + 'contributor.githubHint': 'Make sure your GitHub login matches', + 'contributor.submitResult': 'Submit a result', + 'contributor.firstResults': 'First results', + 'contributor.firstBenchmark': 'First published benchmark on the leaderboard for this hardware', + 'contributor.publishedRuns': 'Published runs', + 'rankings.noSubmissionsFor': 'No submissions yet for', + 'home.figure1': 'Each point is one serving recipe (framework × precision × hardware); larger markers denote best-in-class per vendor.', + 'home.resultsLabel': 'results', 'home.resultLabel': 'result', + 'home.chipsLabel': 'chips', 'home.chipLabel': 'chip', + 'suites.s2.lede': 'Each suite picks a subset of these seven protocols. The metric, direction, and setting are pinned here once. Default scenarios are required for a valid submission; extras are opt-in.', + 'suites.s3.lede': 'One self-contained card per suite. The header pins the primary metric and direction; the workload strip pins the model, hardware budget, precision, and dataset; the protocols row shows which scenarios apply; current leaders surface the top chip per metric.', + 'suites.s4.lede': 'Datasets are content-hash-pinned: once a name is published, the bytes never change. Revising a dataset means a new version (_v2, etc.). Every result is tied to a dataset hash so comparisons stay apples-to-apples across years.', + 'suites.defaultProtocol': 'Default protocol', 'suites.extraProtocol': 'Extra protocol (opt-in)', + 'suites.spec.Metric': 'Metric', 'suites.spec.Direction': 'Direction', + 'suites.spec.Threshold': 'Threshold', 'suites.spec.Cost': 'Cost', + 'suites.spec.Concurrency': 'Concurrency', 'suites.spec.SLA': 'SLA', + 'suites.spec.Arrivals': 'Arrivals', 'suites.spec.Duration': 'Duration', + 'suites.spec.Load': 'Load', 'suites.spec.Streams': 'Streams', + 'suites.spec.Draft model': 'Draft model', 'suites.spec.Mode': 'Mode', + 'suites.spec.Burst': 'Burst', 'suites.spec.Window': 'Window', + 'suites.spec.Direction.Higher is better': 'Higher is better', + 'suites.spec.Direction.Lower is better': 'Lower is better', + 'suites.spec.Direction.Smaller drop is better': 'Smaller drop is better', + 'suites.scenario.accuracy.role': 'Quality gate', + 'suites.scenario.offline.role': 'Peak throughput', + 'suites.scenario.online.role': 'SLA-bound capacity', + 'suites.scenario.interactive.role': 'Single-stream latency', + 'suites.scenario.sustained.role': 'Stability under load', + 'suites.scenario.speculative.role': 'Draft-assisted decode', + 'suites.scenario.burst.role': 'KV pressure', + 'suites.usedBy': 'Used by', 'suites.why': 'Why', + 'suites.dataset': 'Dataset', + 'suites.wlModel': 'Model', 'suites.wlChips': 'Chips', + 'suites.wlPrecision': 'Precision', 'suites.wlDataset': 'Dataset', + 'suites.wlTokens': 'Tokens (in / out)', + 'suite.suite_A.finding.headline': 'Offline winner is not the SLA winner', + 'suite.suite_B.finding.headline': 'Bottom tier is decided by software stack, not VRAM', + 'suite.suite_C.finding.headline': 'Speed without quality is meaningless', + 'suite.suite_D.finding.headline': 'Long context inverts the bandwidth-bound ranking', + 'suite.suite_E.finding.headline': 'Newest flagship has the worst 2× efficiency', + 'suite.suite_F.finding.headline': 'Edge isolates the pure HBM bandwidth ceiling', + 'suite.suite_G.finding.headline': 'Sparse routing rewards bandwidth over FLOPS', + 'suites.prompts': 'Prompts', 'suites.inputP50': 'Input p50', + 'suites.outputP50': 'Output p50', + 'suite.suite_A.title': 'Single-Chip Throughput', + 'suite.suite_A.tagline': 'How fast can one accelerator serve an 8B model?', + 'suite.suite_B.title': 'Multi-Chip Throughput', + 'suite.suite_B.tagline': 'Large-model serving across multiple chips.', + 'suite.suite_C.title': 'Quantization Efficiency', + 'suite.suite_C.tagline': 'Quality-adjusted throughput across precision formats.', + 'suite.suite_D.title': 'Long-Context Inference', + 'suite.suite_D.tagline': '28K-token prefill, compute-bound regime.', + 'suite.suite_E.title': 'Multi-Chip Scaling Efficiency', + 'suite.suite_E.tagline': 'How well does 8B throughput scale to 2 / 4 / 8 chips?', + 'suite.suite_F.title': 'Edge / Consumer Hardware', + 'suite.suite_F.tagline': 'Small models on single-GPU edge hardware.', + 'suite.suite_G.title': 'Mixture-of-Experts (MoE)', + 'suite.suite_G.tagline': 'Sparse routing; bandwidth-bound multi-chip serving.', + 'suite.suite_A.desc': 'The canonical bandwidth-bound regime. 8B Llama on a single accelerator is small enough to fit comfortably in HBM, large enough that decode is memory-bandwidth-bound rather than compute-bound.', + 'suite.suite_B.desc': 'Multi-chip serving of a 70B model. Tests inter-chip communication under throughput load. Bandwidth between accelerators determines scaling efficiency.', + 'suite.suite_C.desc': 'Quantization efficiency on the same 8B model. Compares throughput across BF16, FP8, W8A8, W8A16, and W4A16 on a single chip. Measures speedup vs BF16 and quality efficiency.', + 'suite.suite_D.desc': 'Long-context prefill at ~28K tokens. Pushes arithmetic intensity past the roofline knee, making the workload compute-bound rather than memory-bandwidth-bound.', + 'suite.suite_E.desc': 'Multi-chip scaling from 1 to 8 accelerators. Measures throughput and scaling efficiency at each chip count. Exposes Amdahl effects from communication overhead.', + 'suite.suite_F.desc': 'Edge and consumer GPU regime using Qwen2.5-0.5B. Small model with ~95-token prompts isolates bandwidth on low-memory, low-bandwidth hardware.', + 'suite.suite_G.desc': 'Mixture-of-Experts (MoE) regime with DeepSeek-V2-Lite. Tests sparse routing overhead and aggregate bandwidth utilization at multi-chip scale.', + 'suites.rooflineDetails1': 'A chip optimized for one region, say bandwidth-bound 8B decode, diverges from a chip optimized for another, say compute-bound long-context prefill, as soon as the workload moves. Collapsing heterogeneous workloads into a single composite score hides exactly the trade-offs a buyer needs to see.', + 'suites.rooflineDetails2': 'AccelMark operationalizes spectrum sampling: a set of suites, each anchored to a qualitatively distinct bottleneck region. Bandwidth-bound serving at 8B (A) and 0.5B (F); capacity-then-stack-bound 70B multi-chip (B); the bandwidth-to-compute transition via quantization (C); compute-bound long-context prefill (D); multi-chip communication overhead (E); and sparse MoE routing (G).', + 'suites.rooflineCaption': 'Each suite sits at a different point on the roofline. Bandwidth-bound regimes (left of the knee) reward HBM throughput; compute-bound regimes (right) reward raw FLOPS. A chip ranking changes as the workload moves.', + 'suite.suite_A.finding': 'H200 leads Suite A offline at 5,731 tokens/sec, but on the same suite online tier A100 and A800 sustain 25 queries/sec while H200 caps at 10. The 500 ms p99 TTFT SLA binds well before the throughput ceiling, so the chip that leads offline loses the production tier.', + 'suite.suite_B.finding': 'With 8 chips serving 70B BF16, aggregate VRAM is no longer binding. Ascend 910C ×16 supplies 1,024 GB and still delivers only 723 tokens/sec, below Ascend 910B2 ×8 at 770 tokens/sec. Doubling the hardware buys nothing once vllm-ascend 70B path is the binding constraint.', + 'suite.suite_C.finding': 'Quality efficiency multiplies throughput by accuracy. On A100, W8A8 wins at 3,776 (1.20× speedup, +2 pts accuracy) because INT8 tensor cores engage. FP8 shows zero speedup on A100 because the hardware path is absent and compute falls back to BF16. On H100 the same FP8 column flips to roughly 1.5 to 1.8× speedup.', + 'suite.suite_D.finding': 'Suite D pushes arithmetic intensity past the roofline knee with ~28K-token prefill, making the workload compute-bound rather than memory-bandwidth-bound. Rankings invert relative to Suite A: chips that win on short-prompt decode lose to chips with higher raw FLOPS.', + 'suite.suite_E.finding': 'RTX 4090 D tops the Suite E 2× efficiency leaderboard because its lower per-die throughput keeps the communication share small. H200, the newest flagship, shows the worst NVIDIA 2× efficiency because per-chip throughput grew faster than NVLink 4.0 bandwidth did.', + 'suite.suite_F.finding': 'Suite F uses Qwen2.5-0.5B with ~95-token prompts. Stripping residual compute-path interference exposes raw memory-bandwidth headroom; this is where commodity hardware (RTX 4090, A6000) is most competitive on a per-dollar basis.', + 'suite.suite_G.finding': 'Mixtral activates only 2 of 8 experts per token, keeping arithmetic intensity below dense 8B inference even at 8-chip scale. H20-3e trails A100-40G by ~5% on dense Suite A but beats A100-40G ×8 by 17% on Suite G; its 4,000 GB/s aggregate bandwidth pays off more than its compute.', + 'citation.title': 'Cite the dataset', + 'contributor.firstBenchmark': 'First published benchmark on the leaderboard for this hardware', + }, + cn: { + 'nav.home': '首页', 'nav.results': '结果', 'nav.compare': '对比', 'nav.suites': '套件', + 'nav.community': '社区', 'nav.submit': '提交', 'nav.lang': 'English', + 'nav.contributors': '贡献者', 'nav.wanted': '征集硬件', 'nav.reproduce': '复现任务', + 'footer.tagline': 'AccelMark · 生成属于你的基准用例', + 'footer.cite': '引用数据集', 'footer.api': 'API 索引', + 'footer.source': '源代码', 'footer.discuss': '讨论区', + 'hero.title1': 'AccelMark ', + 'hero.title2': '可复现结果的 LLM 推理性能框架', + 'hero.subtitle': '没有永远的榜首。排名随场景反转——短对话、长上下文、MoE、边缘,各有最佳方案。', + 'hero.tagline': '每条结果公开 result.json、环境指纹、代码哈希、精度报告——不只是一张图上的一个点', + 'hero.reversal': '排名会反转:短对话的赢家,长上下文未必', + 'hero.kpi.benchmarks': '个结果', 'hero.kpi.gpus': '款硬件', 'hero.kpi.vendors': '家厂商', + 'hero.kpi.workloads': '种场景', 'hero.kpi.verified': '已验证', 'hero.kpi.thisWeek': '本周', + 'hero.cta.submit': '提交你的结果 →', 'hero.cta.compare': '对比芯片', 'hero.cta.cite': '引用数据集', + 'hero.chartEyebrow': '排名反转', + 'hero.chartCaption': '同一 vLLM 框架。5090 赢在带宽受限的短对话,A100 赢在计算受限的长上下文。排名反转。', + 'why.eyebrow': '为什么要提交', 'why.title': '跑完就是一条可分享、可引用的公开记录', + 'why.card1.title': '有据可查', 'why.card1.desc': '环境指纹 + 代码哈希 + 精度校验——别人能复现,不是简单的截图', + 'why.card2.title': '抢首发', 'why.card2.desc': '榜单里还没你的卡?去征集硬件看看。
手里正好有对应硬件?领取复现任务', + 'why.card3.title': '能引用', 'why.card3.desc': '论文、报告可以精确引用到某次提交——BibTeX 直达你的 result.json', + 'dist.eyebrow': '01 · 探索', 'dist.title': '方案全景图', + 'dist.subtitle': '一点一方案(框架 × 精度 × 硬件)
同一套件内散得越开,说明可调空间越大', + 'dist.filter.suite': '套件', 'dist.filter.vendor': '厂商', 'dist.filter.framework': '框架', + 'dist.filter.precision': '精度', 'dist.filter.chip': '芯片', 'dist.filter.reset': '↺ 重置', 'dist.all': '全部', + 'dist.metric.label': '指标', 'dist.metric.offline': 'Offline 吞吐', 'dist.metric.online': 'Online 最大 QPS', + 'dist.metric.sustained': '持续吞吐', 'dist.metric.speculative': '投机解码吞吐', + 'dist.tab.byframework': '按框架', 'dist.normalize': '归一化', 'dist.moreviews': '更多视图', + 'dist.info.all': 'Beeswarm — 按套件分组,每点一个方案,大点表示厂商最优
按框架 — 同上,按推理框架分组
Scatter — 吞吐 × QPS,右上角双优
Density — 密度热图,越深方案越多
Heatmap — 套件 × 芯片矩阵,颜色表示吞吐
By Suite — 每套件独立小图
归一化 — Y 轴为列内百分比(0–100%)', + 'dist.info.smallchart': '每套件独立图表', + 'dist.info.showing': '共 {n} 个方案,覆盖 {s} 个套件 · {m}', + 'dist.caption.beeswarm': '每个点都代表一个部署方案,大点表示各厂商最优。Y 轴对数坐标', + 'dist.caption.beeswarm_norm': '每点一个部署方案,大点表示各厂商最优。Y 轴为列内百分比', + 'dist.caption.scatter': '吞吐(x)× 在线 QPS(y),每点一个方案。双轴对数坐标', + 'dist.caption.density': '方案吞吐分布的核密度图,越深提交越多', + 'dist.caption.heatmap': '每个芯片 × 套件的最佳吞吐,颜色深浅表示数值大小', + 'dist.caption.small': '每套件的迷你散点图(吞吐 × QPS)', + 'workloads.eyebrow': '02 · 测试套件', 'workloads.title': '按套件浏览', + 'workloads.subtitle': '每个套件固定模型和测试协议——从单卡吞吐到长上下文服务', + 'workloads.viewfull': '查看全部结果 →', 'workloads.awaiting': '暂无提交', + 'coverage.eyebrow': '03 · 覆盖情况', 'coverage.title': '已测试平台', + 'coverage.subtitle': '方块大小和提交数成正比,颜色区分厂商', + 'recent.eyebrow': '04 · 最新', 'recent.title': '最近提交', 'recent.seeall': '全部 →', + 'community.eyebrow': '05 · 社区', 'community.title': '贡献排行', 'community.all': '全部贡献者 →', + 'community.subtitle': '按已验证提交和复现记录排名——每条结果都链接到完整产物', + 'community.rank': '排名', 'community.contributor': '贡献者', 'community.runs': '提交数', 'community.verified': '已验证', 'community.score': '积分', 'community.empty': '暂无贡献者', + 'contribute.eyebrow': '06 · 参与', 'contribute.title': '准备好了?晒出你的成绩', + 'contribute.body': '硬件在手、框架就绪的话,从零到 PR 合并大约十分钟', + 'contribute.step1': '打开提交向导 — 选套件、平台、框架,它给你拼好命令和目录结构', + 'contribute.step2': '跑一条命令 — 生成完整的 result.jsonenv_info.json、精度报告', + 'contribute.step3': '提 PR — CI 自动校验,合并后生成永久链接供引用', + 'contribute.foot': '第一次接触这套流程?', + 'contribute.guide': '完整流程指引 ↗', + 'contribute.cta1': '打开提交向导 →', 'contribute.cta2': '征集硬件', 'contribute.cta3': '复现任务', + 'rankings.eyebrow': '结果 · 套件 {letter}', + 'rankings.subtitle': '
同一负载,不同框架和硬件的表现差距一目了然', + 'rankings.table.recipe': '方案', 'rankings.table.vendor': '厂商', 'rankings.table.precision': '精度', + 'rankings.table.framework': '框架', 'rankings.table.date': '日期', 'rankings.table.tier': '等级', + 'rankings.table.throughput': '最佳吞吐', 'rankings.table.headroom': '提升空间', + 'rankings.of': '共', 'rankings.results': '条结果', 'rankings.sorted': '排序:', + 'rankings.clear': '清除筛选', 'rankings.showall': '显示全部结果', 'rankings.viewchip': '查看芯片概览', + 'rankings.basket.run': '个', 'rankings.basket.runs': '个', 'rankings.basket.msg': '已加入对比篮', + 'rankings.basket.compare': '开始对比 →', 'rankings.basket.clear': '清空对比篮', + 'rankings.empty': '暂无提交', 'rankings.browse': '浏览结果 →', + 'suites.hero.title': '测试套件', 'suites.hero.sub': '每个套件锁定一种瓶颈——带宽紧缺、算力紧缺、延迟敏感——覆盖推理场景全光谱', + 'suites.hero.cta1': '浏览结果 →', 'suites.hero.cta2': 'GitHub 查看规范', + 'suites.s1.eyebrow': '01 · 方法', 'suites.s1.title': '为什么分套件测试?', + 'suites.s1.lead': '排名会反转。在短对话吞吐场景占优的芯片,可能在长上下文场景落后。每个套件对应一个瓶颈类型——内存受限、计算受限或延迟敏感——让你看清哪个方案在哪里胜出。', + 'suites.s1.body': 'AI 推理负载覆盖了广泛的算力强度区间。Roofline 模型揭示了核心规律:芯片的实际性能取决于内存带宽和计算能力中哪一个成为瓶颈。因为不同负载落在频谱的不同位置,硬件排名不会维持不变。', + 'suites.s2.eyebrow': '02 · 测试场景', 'suites.s2.title': '七种协议,逐个测试', + 'suites.s3.eyebrow': '03 · 详细规格', 'suites.s3.title': '每个套件的具体配置', + 'suites.s4.eyebrow': '04 · 数据集', 'suites.s4.title': '三套固定提示词', + 'compare.eyebrow': '对比', 'compare.title': '逐项对比', + 'compare.sub1': '选择平台开始对比。也可以在', + 'compare.sub2': '结果页', 'compare.sub3': '勾选具体运行来对比框架和精度配置', + 'compare.quickadd': '快速添加', + 'compare.quickaddHint': '每次点击添加该平台的最新运行,选两个以上即可对比', + 'compare.empty': '对比篮为空', + 'submit.eyebrow': '贡献', 'submit.hero': '提交你的测试', + 'contributors.title': '贡献者', + 'contributors.count': '位贡献者', + 'contributors.ranking': '排行', + 'contributors.scoreDesc': '积分 = 运行 + 已验证×3 + 平台×4 + 首发×12 + 工具×5', + 'contributors.rank': '排名', 'contributors.contributor': '贡献者', 'contributors.runs': '运行', + 'contributors.verified': '已验证', 'contributors.platforms': '平台', 'contributors.attribs': '徽章', + 'contributors.score': '积分', 'contributors.empty': '暂无贡献者', + 'contributors.submitFirst': '提交第一个结果', + 'contributors.attribDefs': '徽章说明', + 'chipDetail.notFound': '未找到芯片', + 'chipDetail.backToHome': '返回首页', 'chipDetail.browseResults': '浏览结果', + 'chipDetail.copyLink': '复制链接', 'chipDetail.results': '条结果', + 'chipDetail.bestPerSuiteTitle': '各套件最佳结果', + 'chipDetail.fingerprintTitle': '性能指纹图谱', + 'chipDetail.scalingTitle': '多卡扩展收益如何?', + 'chipDetail.peersTitle': '同级芯片对比', + 'chipDetail.thSuite': '套件', 'chipDetail.thChips': '芯片', + 'chipDetail.thFramework': '框架', 'chipDetail.thPrecision': '精度', + 'chipDetail.thDate': '日期', 'chipDetail.thSubmitter': '提交者', + 'chipDetail.thTier': '等级', + 'chipDetail.thPrimaryMetric': '主要指标', + 'chipDetail.suiteUnit': '个套件', 'chipDetail.suiteUnitPlural': '个套件', + 'chipDetail.runUnit': '条运行', 'chipDetail.runUnitPlural': '条运行', + 'chipDetail.frameworkUnit': '个框架', 'chipDetail.frameworkUnitPlural': '个框架', + 'chipDetail.precisionUnit': '种精度', 'chipDetail.precisionUnitPlural': '种精度', + 'chipDetail.chipCountVariants': '种芯片规格', + 'chipDetail.deployedAt': '部署于', + 'chipDetail.compareConfigs': '对比此芯片的配置', + 'chipDetail.browseVendor': '浏览', + 'chipDetail.copyLinkTitle': '复制芯片概览链接', + 'chipDetail.bestPerSuiteEyebrow': '各套件最佳', + 'chipDetail.bestPerSuiteSub': '每个基准套件提交的最高结果。', + 'chipDetail.bestScoreCountTitle': '各套件最佳分数统计', + 'chipDetail.ofBest': '较最佳', + 'chipDetail.globalBest': '全局最佳', + 'chipDetail.rankedInSuite': '在套件中排名', + 'chipDetail.noSubmission': '无提交', + 'chipDetail.noSubmissionAt': '该芯片数量下无提交。', + 'chipDetail.notSubmitted': '未提交', + 'chipDetail.openRun': '打开运行', + 'chipDetail.openRunDetails': '打开运行详情', + 'chipDetail.cmdClickAllInSuite': 'Cmd/Ctrl + 点击查看此套件全部结果。', + 'chipDetail.everySubEyebrow': '次提交', + 'chipDetail.runsOnFile': '条运行记录', + 'chipDetail.runsOnFileSub': '所有套件的全部提交,按日期排序。', + 'chipDetail.downloadRadarTitle': '下载雷达图为 PNG', + 'chipDetail.downloadScalingTitle': '下载扩展图为 PNG', + 'chipDetail.png': 'PNG', + 'chipDetail.flashCopied': '已复制!', + 'chipDetail.flashCopyFailed': '复制失败', + 'chipDetail.flashFailed': '下载失败', + 'chipDetail.flashSaved': '已保存!', + 'chipDetail.fingerprintEyebrow': '指纹', + 'chipDetail.fingerprintSub1': '每轴为一个基准套件。离中心越远 = 吞吐越高。在某一负载上称霸的芯片可能在另一负载上落后。', + 'chipDetail.fingerprintSubMissing': '无提交的套件收缩至中心({missing})。', + 'chipDetail.radarAriaLabel': '{chip} 在所有套件中的性能雷达图', + 'chipDetail.fpLegendAria': '指纹雷达图的套件图例', + 'chipDetail.scalingEyebrow': '扩展', + 'chipDetail.scalingSub': '单卡吞吐随芯片数量变化。理想情况下线性增长;实际负载中收益递减。', + 'chipDetail.scalingChartAria': '单卡扩展图表', + 'chipDetail.sclCellTitle': '各套件各芯片数量扩展分解', + 'chipDetail.sclBreakdownAria': '扩展效率表', + 'chipDetail.sclLegendAria': '扩展图表的指标图例', + 'chipDetail.peersEyebrow': '同级', + 'chipDetail.peersSub': '规格相近的芯片——同厂商、同显存或同代。', + 'chipDetail.cardTitle': '至少有一次提交的套件', + 'chipDetail.stale': '此芯片可能已被重命名或从数据集中移除。', + 'rankings.clearFilters': '清除筛选', + 'submit.heroSub': '从硬件检测到可发布的引用结果。', + 'submit.flow_detect': '检测', 'submit.flow_run': '运行', + 'submit.flow_validate': '验证', 'submit.flow_preview': '预览', + 'submit.flow_submit': '提交', 'submit.flow_published': '已发布', + 'submit.step_hardware': '硬件', 'submit.step_suite': '套件', + 'submit.step_run': '运行', 'submit.step_submit': '提交', + 'submit.chooseHardware': '选择硬件', + 'submit.searchCatalog': '搜索列表', 'submit.vendor': '厂商', + 'submit.gpuAccelerator': 'GPU / 加速卡', 'submit.chipCount': '卡数', + 'submit.pickSuite': '选择套件', 'submit.runValidate': '运行 & 验证', + 'submit.install': '安装', 'submit.copy': '复制', + 'submit.benchmark': '运行测试', 'submit.validateLocally': '本地验证', + 'submit.submitForPublication': '提交发布', + 'submit.submitForPublicationSub': '每条合并结果链接到 result.json、env_info.json 和复现指引', + 'submit.prRecommendedSub': 'Fork → 运行测试 → 提 PR', + 'submit.uploadIssueSub': '附上 result.json + env_info.json', + 'submit.shareDiscussionsSub': '展示交流 — 硬件技巧、优化笔记', + 'submit.searchPlaceholder': '如 H100、MI350、昇腾 910C、RTX 5090 等', + 'submit.chooseHardwareSub': '完整列表 — 数据中心、工作站、消费级', + 'submit.pickSuiteHint': '新贡献者建议从 Suite A 或 Suite F 开始(约10分钟)', + 'submit.firstBenchmark': '数据中心 GPU 首次测试约 11 分钟', + 'submit.customName': '或自定义名称', 'submit.customPlaceholder': '如不在上述列表中', + 'submit.artifactsPerRun': '每次运行产出:', + 'submit.done': '完成', 'submit.estimatedRuntime': '预计耗时', + 'submit.suggestedPrefix': '建议', 'submit.recommended': '推荐', + 'submit.suiteHint.A': '最佳入门基准 · 单卡推理', + 'submit.suiteHint.B': '大模型多卡', + 'submit.suiteHint.C': '量化效率', + 'submit.suiteHint.D': '长上下文推理', + 'submit.suiteHint.E': '多卡扩展', + 'submit.suiteHint.F': '边缘/消费级 GPU · 离线+在线', + 'submit.suiteHint.G': 'MoE 多卡', + 'submit.flashCopied': '已复制!', 'submit.flashFailed': '失败', + 'submit.prRecommended': 'Pull Request(推荐)', + 'submit.uploadIssue': '通过 GitHub Issue 上传', + 'submit.shareDiscussions': '在 Discussions 分享', + 'submit.back': '上一步', 'submit.next': '下一步', + 'compare.stalePrefix': '此对比链接指向的', + 'compare.aRun': '条运行', 'compare.runs': '条运行', + 'compare.staleMid': '这些运行', + 'compare.staleSuffix': '已不存在于数据集中。', + 'compare.comparing': '对比中', 'compare.stale': '过期', 'compare.run': '运行', + 'compare.clearStartOver': '清空重来', + 'compare.quickAddPlatform': '快速添加平台', + 'compare.quickAddPlatformHint': '点击任意芯片将其最佳运行加入对比篮。', + 'compare.clearAll': '全部清除', 'compare.suite': '套件', + 'compare.noSubmissions': '所选平台暂无提交数据。', + 'compare.metric': '指标', 'compare.headToHeadCharts': '逐项图表对比', + 'compare.copyShareLink': '复制分享链接', + 'compare.copyShareLinkTitle': '复制此对比的分享链接', + 'compare.downloadChartTitle': '下载图表为 PNG', + 'compare.png': 'PNG', + 'compare.flashCopied': '已复制!', + 'compare.flashCopyFailed': '复制失败', + 'compare.flashFailed': '下载失败', + 'compare.flashSaved': '已保存!', + 'compare.across': '覆盖', + 'compare.addTo': '加入', + 'compare.chipCountVariants': '种芯片规格', + 'compare.clickTo': '点击', + 'compare.compareBasketCtx': '对比篮;Cmd/Ctrl+点击可打开芯片总览。', + 'compare.removeFrom': '移出', + 'compare.submissionUnit': '次提交', + 'compare.submissionUnitPlural': '次提交', + 'compare.suiteUnit': '个套件', + 'compare.suiteUnitPlural': '个套件', + 'compare.higherIsBetter': '越高越好', + 'compare.lowerIsBetter': '越低越好', + 'compare.chartsHintOf': '共', 'compare.chartsHintChips': '芯片', + 'compare.chartSingleChipOffline': '单卡离线吞吐', + 'compare.chartTokByConcurrency': '各并发度吞吐', + 'compare.noSuiteData': '所选芯片在该套件无数据。', + 'compare.noSuiteData1': '所选芯片在 ', + 'compare.noSuiteData2': ' 中无数据。', + 'compare.try': '试试 ', + 'compare.instead': '', + 'community.entries': '条', + 'community.of': ',共', + 'compare.chartsHintAll': '所选芯片在该套件均有数据。', + 'compare.chartFailed': '图表渲染失败。', + 'compare.chartEdgeOffline': '边缘离线吞吐', + 'compare.chartMoeOffline': 'MoE 离线吞吐', + 'compare.chartMultiChipTotal': '多卡总吞吐', + 'compare.chartPeakOffline': '峰值离线吞吐', + 'compare.chartTokensPerSec': 'tokens/秒', + 'compare.chartConcurrency': '并发数', + 'compare.chartPerChipThroughput': '单卡吞吐', + 'compare.chartPerChipSub': '总吞吐 ÷ 卡数;越高 = 单卡效率越好。', + 'compare.chartTokPerSecPerChip': 'tokens/秒/卡', + 'compare.chartQuantThroughput': '各精度吞吐对比', + 'compare.chartQuantSub': 'BF16 vs 量化格式(FP8、INT8、INT4)。', + 'compare.chartPrecision': '精度', + 'compare.chartTtftP50': 'TTFT p50', 'compare.chartTtftP90': 'TTFT p90', 'compare.chartTtftP99': 'TTFT p99', + 'compare.chartTpotP50': 'TPOT p50', 'compare.chartTpotP90': 'TPOT p90', 'compare.chartTpotP99': 'TPOT p99', + 'compare.chartLongContextLatency': '长上下文延迟', + 'compare.chartLongContextSub': '28K token 下的 TTFT / TPOT 百分位。', + 'compare.chartLatency': '延迟', + 'compare.chartThroughputByCount': '各卡数吞吐', + 'compare.chartThroughputByCountSub': '总 tokens/秒随卡数的扩展情况。', + 'compare.chartScalingEfficiency': '扩展效率', + 'compare.chartScalingEfficiencySub': '2× 效率 = (双卡吞吐 / 单卡吞吐 × 2) × 100。', + 'compare.chartLinearIdeal': '线性理想 (100%)', + 'compare.chartGood80': '良好 (80%)', + 'compare.chartEfficiencyPct': '效率 %', + 'wanted.title': '征集硬件', + 'wanted.desc1': '已收录但尚未有测试结果的加速卡', + 'wanted.open': '个待测', 'wanted.covered': '个已覆盖', + 'wanted.desc2': '首个发布该平台结果的贡献者将在', + 'wanted.contribIndex': '贡献者榜单', + 'wanted.filterVendor': '厂商', 'wanted.filterTier': '级别', 'wanted.filterSearch': '搜索', + 'wanted.searchPlaceholder': '平台名称', + 'wanted.thPlatform': '平台', 'wanted.thVendor': '厂商', 'wanted.thTier': '级别', + 'wanted.thMemory': '显存', 'wanted.thSuites': '推荐套件', + 'wanted.submit': '提交', 'wanted.allCovered': '所有平台均已覆盖。', + 'wanted.allVendors': '全部厂商', 'wanted.allTiers': '全部级别', + 'wanted.datacenter': '数据中心', 'wanted.cloud': '云端', + 'wanted.workstation': '工作站', 'wanted.consumer': '消费级', 'wanted.edge': '边缘', + 'reproduce.title': '复现任务', + 'reproduce.desc': '独立复现社区提交且误差在5%以内,双方均可获得验证者积分。在PR和 meta.reproduces_run_id 中引用原始 run_id。', + 'reproduce.whyTitle': '为什么有这些任务?', 'reproduce.verified': '已验证', + 'reproduce.verifiedTitle': '交叉验证覆盖', 'reproduce.awaitingTitle': '等待验证', + 'reproduce.open': '待验证', 'reproduce.procedure': '操作流程', + 'reproduce.thOriginal': '原始提交(社区)', 'reproduce.thVerified': '验证复现', + 'reproduce.thSuite': '套件', 'reproduce.thConfirmed': '确认日期', + 'reproduce.btnOriginal': '原始', 'reproduce.btnVerified': '验证', + 'reproduce.btnDetails': '详情', 'reproduce.btnRequest': '请求', + 'reproduce.quest': '个任务', 'reproduce.quests': '个任务', + 'reproduce.pair': '对', 'reproduce.pairs': '对', + 'reproduce.record': '条', 'reproduce.records': '条', + 'reproduce.filterVendor': '厂商', + 'reproduce.filterSuite': '套件', + 'reproduce.filterSearch': '搜索', + 'reproduce.searchPlaceholder': '平台、方案、提交者', + 'reproduce.allVendors': '全部厂商', + 'reproduce.allSuites': '全部套件', + 'reproduce.thPlatform': '平台', + 'reproduce.thRecipe': '方案', + 'reproduce.thSubmitter': '提交者', + 'reproduce.thDate': '日期', + 'reproduce.thTier': '等级', + 'reproduce.empty': '暂无开放的复现任务。', + 'reproduce.statOpen': '{n} 个开放{w}', + 'reproduce.statLinked': '{n} 条交叉验证{w}', + 'reproduce.statPairs': '{np} 个平台-套件{pw}等待验证 · {nv} 对已验证覆盖', + 'reproduce.emptyVerifiedNote': '验证运行的 result.json 存储了 meta.reproduces_run_id,在下一次 generate.py 刷新后自动填充此表。', + 'reproduce.countLine': '个社区运行,所在平台尚无已验证覆盖。', + 'reproduce.emptyVerified': '暂无交叉验证记录。', + 'reproduce.criteriaLead': '开放列表是全自动生成的——当以下两个条件同时满足时,该行即出现:', + 'reproduce.criteriaRule1': '该提交为社区级别(已发布,尚未独立验证)。', + 'reproduce.criteriaRule2': '同一硬件平台+测试套件尚无已验证结果。', + 'reproduce.criteriaNote': '交叉验证覆盖是独立的:仅列出 meta.reproduces_run_id 明确引用原始社区 run_id 的运行。该链接在提交时设定,不会根据芯片或套件自动推断。', + 'reproduce.countLine': '个社区运行,所在平台尚无已验证覆盖。', + 'reproduce.procedure1': '选择一个任务,记下其 run_id。', + 'reproduce.procedure2': '在匹配的硬件上重新运行同一套件,加上 --reproduces-run-id 参数。', + 'reproduce.procedure3': '提交 Pull Request,合并后记录即出现在下方。', + 'reproduce.procedure4': '双方均可在贡献者榜单获得积分。', + 'badge.first-result.label': '首发', 'badge.pioneer.label': '先驱', + 'badge.verified.label': '已验证', 'badge.verifier.label': '验证者', + 'badge.prolific.label': '高产', 'badge.multi-chip.label': '多平台', + 'badge.multi-vendor.label': '跨厂商', 'badge.runner.label': '工具作者', + 'badge.first-result.desc': '首个发布某硬件平台的基准测试结果', + 'badge.pioneer.desc': '榜单前三位贡献者之一', + 'badge.verified.desc': '至少有一条已验证级别结果', + 'badge.verifier.desc': '独立复现并发布为已验证', + 'badge.prolific.desc': '发布十条或以上基准测试', + 'badge.multi-chip.desc': '在三个或以上不同硬件平台上有结果', + 'badge.multi-vendor.desc': '在两个或以上厂商平台上有结果', + 'badge.runner.desc': '贡献了一个 runner 实现', + 'suites.readMore': '展开阅读', + 'suites.showLess': '收起', 'suites.concreteFinding': '实测发现', + 'suites.currentLeaders': '当前领先', 'suites.openResults': '查看结果 →', + 'suites.viewJson': '查看 suite.json', 'suites.coverage': '覆盖情况', + 'suites.protocols': '测试协议', + 'suites.extra': '额外', + 'suites.inversion.A.eyebrow': '离线 vs 在线', + 'suites.inversion.A.title': '吞吐冠军输掉了 SLA 对决', + 'suites.inversion.A.body': 'Suite A:H200 离线测试以 5,731 tokens/sec 领先,但一旦施加 500 ms p99 TTFT SLA,其吞吐被限制在 10 queries/sec。同一硬件级别的 A100 和 A800 稳定维持 25 queries/sec。离线和在线并非同一场比赛。', + 'suites.inversion.G.eyebrow': '密集 vs MoE', + 'suites.inversion.G.title': 'H20-3e:密集落后 5%,MoE 领先 17%', + 'suites.inversion.G.body': '在密集的 Suite A 上,H20-3e 落后 A100-40G 约 5%。但在 Suite G 的稀疏 Mixtral 路由中,同一芯片反超 A100-40G ×8 达 17%。稀疏激活使算力强度低于密集 8B,因此带宽(而非 FLOPS)成为性能上限。', + 'suites.inversion.E.eyebrow': '多卡 Amdahl 效应', + 'suites.inversion.E.title': '最新旗舰的 2× 扩展效率最差', + 'suites.inversion.E.body': 'RTX 4090 D 登顶 Suite E 2× 扩展效率榜首,正是因为其单卡吞吐低,通信时间占比小。H200 的单卡吞吐增长速度超过了 NVLink 4.0 带宽的提升,导致其 2× 扩展效率落在 NVIDIA 系列底部。', + 'suites.scenario.accuracy.desc': '基于套件基线模型的 MMLU 子集评分。在任何吞吐场景之前运行;精度下降超过阈值的芯片,该套件所有其他数据均无效。', + 'suites.scenario.offline.desc': '所有请求同时提交,无 SLA 限制,无并发上限。纯粹的峰值能力数据,确立芯片在该负载下的天花板。', + 'suites.scenario.online.desc': '在 Poisson 请求到达下扫描请求负载,报告仍能满足 500 ms p99 TTFT SLA 的最高 queries/sec。这是生产流量真正需要遵守的指标。', + 'suites.scenario.interactive.desc': '一次仅处理一个请求,无并发。聊天窗口 UX 基准;队列延迟最小,主要受解码延迟和软件开销影响。', + 'suites.scenario.sustained.desc': '固定并发负载持续 15 至 30 分钟。报告首尾 60 秒窗口之间的节流比,暴露热节流和内存碎片问题。', + 'suites.scenario.speculative.desc': '离线负载,同时加载 1B 草稿模型辅助目标模型解码。报告投机解码接受率和端到端加速比;帮助你判断投机解码是否值得消耗显存。', + 'suites.scenario.burst.desc': '交替施加 5× 稳定到达率(短脉冲窗口)与正常流量,报告突发期间的 TTFT p99。重点考验 KV 缓存、准入控制和预热路径。', + 'suites.dataset.sharegpt_standard_v1.notes': '精选自生产级 LLM-API 流量;token 长度 p99 ≈ 2,100。', + 'suites.dataset.sharegpt_longctx_v1.notes': '拼接多轮对话,将预填充推过 roofline 拐点。', + 'suites.dataset.sharegpt_edge_v1.notes': '短单轮提示词;使边缘套件保持带宽隔离。', + 'suites.s5.eyebrow': '05 · 扩展', 'suites.s5.title': '提交新套件方案', + 'suites.s5.body': '有 AccelMark 尚未覆盖的工作负载场景——长上下文推理、投机解码经济学、领域微调模型?在 Discussions 提交一页方案草图,描述瓶颈区域和参考 SLA。贡献流程与提交新结果相同。', + 'suites.s5.cta': '提交方案 →', + 'suites.noSubmissions': '暂无符合条件的结果', + 'cite.title': '引用数据集', 'cite.copyBibtex': '复制 BibTeX', + 'cite.copyMarkdown': '复制 Markdown', 'cite.apiTitle': '机器可读 API', + 'cite.lead': '三级引用体系——框架级、版本快照级、单项可复现运行级。所有社区结果均遵循 CC BY 4.0 协议。', + 'cite.singleResultDesc': '在结果页打开任意运行 → 使用详情面板中的复制链接复制 Markdown复制 BibTeX。每条运行都链接到 result.json、运行器哈希和复现脚本。', + 'cite.browseResults': '浏览结果 →', + 'cite.apiFuture': '即将推出:/api/runs.json/api/runs/<run_id>.json、版本快照、每版本 Zenodo DOI。', + 'cite.projectDesc': '引用 AccelMark 基准测试框架和方法论。', + 'contributor.githubHint': '请确保 GitHub 用户名匹配', + 'contributor.submitResult': '提交结果', + 'contributor.firstResults': '首发结果', + 'contributor.firstBenchmark': '该硬件在排行榜上的首个公开基准测试', + 'contributor.publishedRuns': '已发布运行', + 'rankings.noSubmissionsFor': '暂无提交记录', + 'home.figure1': '每个数据点代表一个推理方案(框架 × 精度 × 硬件);较大的标记代表各厂商最佳。', + 'home.resultsLabel': '条结果', 'home.resultLabel': '条结果', + 'home.chipsLabel': '款芯片', 'home.chipLabel': '款芯片', + 'suites.s2.lede': '每个套件从七种协议中选择一个子集。指标、方向和参数在此统一定义。套件卡片只需标注适用哪些。默认场景为必选;额外场景供厂商进一步表征特定场景。', + 'suites.s3.lede': '每个套件一张独立卡片。头部标注主要指标和方向;参数区固定模型、硬件、精度和数据集;协议行显示适用场景;当前领先区展示各指标最佳芯片。', + 'suites.s4.lede': '数据集通过内容哈希锁定:一旦发布,内容不可更改。修订数据集意味着新版本(_v2 等)。每条结果绑定数据集哈希,确保跨年度对比的一致性。', + 'suites.defaultProtocol': '默认协议', 'suites.extraProtocol': '额外协议(可选)', + 'suites.spec.Metric': '指标', 'suites.spec.Direction': '方向', + 'suites.spec.Threshold': '阈值', 'suites.spec.Cost': '耗时', + 'suites.spec.Concurrency': '并发', 'suites.spec.SLA': 'SLA', + 'suites.spec.Arrivals': '请求模式', 'suites.spec.Duration': '持续时长', + 'suites.spec.Load': '负载', 'suites.spec.Streams': '流数', + 'suites.spec.Draft model': '草稿模型', 'suites.spec.Mode': '模式', + 'suites.spec.Burst': '突发', 'suites.spec.Window': '窗口', + 'suites.spec.Direction.Higher is better': '越高越好', + 'suites.spec.Direction.Lower is better': '越低越好', + 'suites.spec.Direction.Smaller drop is better': '降幅越小越好', + 'suites.scenario.accuracy.role': '质量把关', + 'suites.scenario.offline.role': '峰值吞吐', + 'suites.scenario.online.role': 'SLA 容量', + 'suites.scenario.interactive.role': '单流延迟', + 'suites.scenario.sustained.role': '负载稳定性', + 'suites.scenario.speculative.role': '投机解码', + 'suites.scenario.burst.role': 'KV 压力测试', + 'suites.usedBy': '使用场景', 'suites.why': '说明', + 'suites.dataset': '数据集', + 'suites.wlModel': '模型', 'suites.wlChips': '芯片数', + 'suites.wlPrecision': '精度', 'suites.wlDataset': '数据集', + 'suites.wlTokens': 'Token(输入/输出)', + 'suite.suite_A.finding.headline': '离线冠军并非 SLA 赢家', + 'suite.suite_B.finding.headline': '底层取决于软件栈,而非显存', + 'suite.suite_C.finding.headline': '脱离质量的加速毫无意义', + 'suite.suite_D.finding.headline': '长上下文反转带宽排名', + 'suite.suite_E.finding.headline': '最新旗舰 2× 扩展效率最差', + 'suite.suite_F.finding.headline': '边缘场景揭示纯粹 HBM 带宽上限', + 'suite.suite_G.finding.headline': '稀疏路由奖励带宽而非 FLOPS', + 'suites.prompts': '提示词数', 'suites.inputP50': '输入 p50', + 'suites.outputP50': '输出 p50', + 'suite.suite_A.title': '单卡吞吐', + 'suite.suite_A.tagline': '单卡能跑多快?8B 模型推理速度测试。', + 'suite.suite_B.title': '多卡吞吐', + 'suite.suite_B.tagline': '70B大模型多卡推理。', + 'suite.suite_C.title': '量化效率', + 'suite.suite_C.tagline': '不同精度格式下,质量加权的吞吐对比。', + 'suite.suite_D.title': '长上下文推理', + 'suite.suite_D.tagline': '28K token 预填充,计算受限场景。', + 'suite.suite_E.title': '多卡扩展效率', + 'suite.suite_E.tagline': '8B 吞吐如何随 2/4/8 卡扩展?', + 'suite.suite_F.title': '边缘/消费级硬件', + 'suite.suite_F.tagline': '小模型在单卡边缘硬件上的表现。', + 'suite.suite_G.title': '混合专家(MoE)模型', + 'suite.suite_G.tagline': '稀疏路由;带宽受限的多卡推理。', + 'suite.suite_A.desc': '经典的带宽受限场景。8B Llama 在单卡上运行——小到可以舒适装入 HBM,大到解码阶段受限于内存带宽而非计算能力。这是大多数 LLM 基准测试锚定的核心推理场景。', + 'suite.suite_B.desc': '70B 大模型多卡推理。测试吞吐负载下的卡间通信开销。加速卡之间的互联带宽决定了扩展效率。', + 'suite.suite_C.desc': '同一 8B 模型的量化效率测试。在单卡上对比 BF16、FP8、W8A8、W8A16 和 W4A16 五种精度的吞吐表现,衡量相对于 BF16 的加速比和质量效率。', + 'suite.suite_D.desc': '~28K token 长上下文预填充。将算力强度推过 roofline 拐点,使负载从内存带宽受限变为计算受限。', + 'suite.suite_E.desc': '从 1 卡到 8 卡的多卡扩展测试。衡量每个卡数下的吞吐量和扩展效率,暴露通信开销带来的 Amdahl 效应。', + 'suite.suite_F.desc': '边缘和消费级 GPU 场景,使用 Qwen2.5-0.5B。小模型配合 ~95 token 的短提示,在低显存、低带宽硬件上精准测试内存带宽。', + 'suite.suite_G.desc': 'Mixture-of-Experts(MoE)场景,使用 DeepSeek-V2-Lite。测试稀疏路由开销和多卡聚合带宽利用率。', + 'suites.rooflineDetails1': '针对某一区域优化的芯片——比如带宽受限的 8B 解码——与针对另一区域优化的芯片——比如计算受限的长上下文预填充——一旦负载移动,表现就出现分化。将异构负载压缩为单一综合评分,恰好掩盖了采购方最需要的权衡信息。', + 'suites.rooflineDetails2': 'AccelMark 实施了频谱采样:一组套件,每个套件锚定一个性质上截然不同的瓶颈区域。8B (A) 和 0.5B (F) 的带宽受限推理;70B 多卡 (B) 的容量受限推理;通过量化实现的带宽到计算过渡 (C);计算受限的长上下文预填充 (D);多卡通信开销 (E);以及稀疏 MoE 路由 (G)。', + 'suites.rooflineCaption': '每个套件位于 roofline 图上的不同位置。带宽受限区域(拐点左侧)奖励 HBM 吞吐;计算受限区域(右侧)奖励原始 FLOPS。芯片的排名会随负载移动而改变。', + 'suite.suite_A.finding': 'H200 在 Suite A 离线测试中以 5,731 tokens/sec 领先,但在同一套件的在线测试中,A100 和 A800 稳定维持 25 queries/sec,而 H200 仅达 10。500 ms p99 TTFT SLA 在吞吐触及天花板之前就已生效,因此离线领先的芯片在线上反而落败。', + 'suite.suite_B.finding': '8 卡推理 70B BF16 时,总显存不再是瓶颈。Ascend 910C ×16 提供 1,024 GB 显存,却仅输出 723 tokens/sec,低于 Ascend 910B2 ×8 的 770 tokens/sec。一旦 vllm-ascend 的 70B 路径成为约束条件,翻倍硬件也毫无收益。', + 'suite.suite_C.finding': '质量效率 = 吞吐 × 精度。在 A100 上,W8A8 以 3,776 胜出(1.20× 加速,精度+2),因为 INT8 tensor core 得到了利用。FP8 在 A100 上毫无加速——硬件路径缺失,计算回退到 BF16。而在 H100 上,同一 FP8 列反转为约 1.5 到 1.8× 加速。', + 'suite.suite_D.finding': 'Suite D 通过 ~28K token 预填充将算力强度推过 roofline 拐点,使负载从内存带宽受限变为计算受限。排名相对于 Suite A 反转:短提示解码中胜出的芯片,在拥有更高原始 FLOPS 的芯片面前败下阵来。', + 'suite.suite_E.finding': 'RTX 4090 D 登顶 Suite E 2× 扩展效率榜首,正是因为其单卡吞吐较低,通信时间占比小。H200 作为最新旗舰,其 2× 扩展效率却是 NVIDIA 系列中最差的——因为单卡吞吐的增长速度超过了 NVLink 4.0 带宽的提升。', + 'suite.suite_F.finding': 'Suite F 使用 Qwen2.5-0.5B 配合 ~95 token 提示。剥离残余计算路径的干扰后,暴露出纯粹的内存带宽余量;这正是消费级硬件(RTX 4090、A6000)在性价比上最具竞争力的场景。', + 'suite.suite_G.finding': 'Mixtral 每个 token 仅激活 8 个专家中的 2 个,即使在 8 卡规模下,算力强度仍低于密集 8B 推理。H20-3e 在密集 Suite A 上落后 A100-40G 约 5%,但在 Suite G 上反超 A100-40G ×8 达 17%;其 4,000 GB/s 聚合带宽带来的收益超过了算力差距。', + 'citation.title': '引用数据集', + 'contributor.firstBenchmark': '该硬件平台在榜单上的首次发布', + } +}); diff --git a/leaderboard/site/assets/js/main.js b/leaderboard/site/assets/js/main.js index 5750f87..a2b55c7 100644 --- a/leaderboard/site/assets/js/main.js +++ b/leaderboard/site/assets/js/main.js @@ -1,7 +1,7 @@ // main.js — entry point. Wires up router, data, and the top nav. import { init as initData } from "./data.js"; -import { mount, register, start } from "./router.js"; +import { mount, register, start, refresh } from "./router.js"; import { initModal } from "./modal.js"; import { render as renderHome } from "./views/home.js"; import { render as renderRankings } from "./views/rankings.js"; @@ -35,6 +35,9 @@ function boot() { register("/reproduce", renderReproduce); start(); + + // Expose router refresh for i18n toggle + window.router = { refresh }; } if (document.readyState === "loading") { diff --git a/leaderboard/site/assets/js/router.js b/leaderboard/site/assets/js/router.js index 924fa40..27b6105 100644 --- a/leaderboard/site/assets/js/router.js +++ b/leaderboard/site/assets/js/router.js @@ -134,6 +134,8 @@ export function start() { else dispatch(); } +export function refresh() { dispatch(); } + // ── Compare basket (shared state across views) ── const basketListeners = new Set(); diff --git a/leaderboard/site/assets/js/views/chip-detail.js b/leaderboard/site/assets/js/views/chip-detail.js index ba813f3..591374d 100644 --- a/leaderboard/site/assets/js/views/chip-detail.js +++ b/leaderboard/site/assets/js/views/chip-detail.js @@ -28,6 +28,7 @@ import { esc, fmtDate, shortVersion, submitterHandle, copyToClipboard, flashButtonLabel, downloadCanvasAsPng, } from "../utils.js"; +const _i = (k, r) => (window._i ? window._i(k, r) : k); export function render({ el, params }) { const slug = params.slug; @@ -40,11 +41,11 @@ export function render({ el, params }) { el.innerHTML = `
-

No chip found for ${esc(slug)}.

-

It may have been removed, or the link is from an older revision of the dataset.

+

${_i('chipDetail.notFound')} ${esc(slug)}.

+

${_i('chipDetail.stale')}

- Back to home - Browse results + ${_i('chipDetail.backToHome')} + ${_i('chipDetail.browseResults')}
`; @@ -79,17 +80,15 @@ export function render({ el, params }) { const memoryStr = sample.memory_gb ? `${sample.memory_gb} GB` : ""; const factPills = [ - `${activeSuites.length} suite${activeSuites.length === 1 ? "" : "s"}`, - `${rs.length} run${rs.length === 1 ? "" : "s"}`, - `${frameworks.size} framework${frameworks.size === 1 ? "" : "s"}`, - `${precisions.size} precision${precisions.size === 1 ? "" : "s"}`, + `${activeSuites.length} ${_i('chipDetail.suiteUnit', activeSuites.length === 1 ? 'suite' : 'suites')}`, + `${rs.length} ${_i('chipDetail.runUnit', rs.length === 1 ? 'run' : 'runs')}`, + `${frameworks.size} ${_i('chipDetail.frameworkUnit', frameworks.size === 1 ? 'framework' : 'frameworks')}`, + `${precisions.size} ${_i('chipDetail.precisionUnit', precisions.size === 1 ? 'precision' : 'precisions')}`, ]; - // Chip-count fact only adds noise for single-variant chips; only - // surface it when the chip has been deployed at >1 fan-out. if (chipCounts.length > 1) { - factPills.push(`${chipCounts.length} chip-count variants (${chipCounts.map((c) => `×${c}`).join(", ")})`); + factPills.push(`${chipCounts.length} ${_i('chipDetail.chipCountVariants', 'chip-count variants')} (${chipCounts.map((c) => `×${c}`).join(", ")})`); } else if (chipCounts.length === 1 && chipCounts[0] > 1) { - factPills.push(`deployed at ×${chipCounts[0]}`); + factPills.push(`${_i('chipDetail.deployedAt')} ×${chipCounts[0]}`); } el.innerHTML = ` @@ -102,15 +101,15 @@ export function render({ el, params }) {

${factPills.map(esc).join(" · ")}

${latestRid - ? `Compare configurations` + ? `${_i('chipDetail.compareConfigs')}` : ""} - Browse ${esc(sample.vendor)} results + ${_i('chipDetail.browseVendor')} ${esc(sample.vendor)} ${_i('chipDetail.results')}
@@ -118,10 +117,10 @@ export function render({ el, params }) {
- 01 · Best per suite -

Best result per suite

+ ${_i('chipDetail.bestPerSuiteEyebrow')} +

${_i('chipDetail.bestPerSuiteTitle')}

-

Top primary-metric run in each suite. Click a card to open its details.

+

${_i('chipDetail.bestPerSuiteSub')}

${SUITE_ORDER.map((sid) => renderSuiteCard(sid, bestPerSuite.get(sid), slug)).join("")} @@ -135,10 +134,10 @@ export function render({ el, params }) {
- ${runsNum} · Every submission -

${rs.length} run${rs.length === 1 ? "" : "s"} on file

+ ${runsNum}${_i('chipDetail.everySubEyebrow')} +

${rs.length} ${_i('chipDetail.runsOnFile', rs.length === 1 ? 'run on file' : 'runs on file')}

-

Sorted newest first. Click a row to open the run detail.

+

${_i('chipDetail.runsOnFileSub')}

${renderRunsTable(rs)} @@ -211,13 +210,13 @@ async function _downloadChipChart(btn) { const wrap = btn.closest(".chip-fp-canvas, .chip-scl-canvas"); const canvas = wrap && wrap.querySelector("canvas"); if (!canvas) { - flashButtonLabel(btn, "Failed", { holdMs: 2000, className: "is-failed", labelSelector: ".chart-dl-btn-label" }); + flashButtonLabel(btn, _i('chipDetail.flashFailed'), { holdMs: 2000, className: "is-failed", labelSelector: ".chart-dl-btn-label" }); return; } const slug = _activeChipSlug(); const filename = `${slug}-${kind === "radar" ? "fingerprint" : "scaling"}.png`; const ok = await downloadCanvasAsPng(canvas, { filename }); - flashButtonLabel(btn, ok ? "Saved" : "Failed", { + flashButtonLabel(btn, ok ? _i('chipDetail.flashSaved') : _i('chipDetail.flashFailed'), { holdMs: ok ? 1400 : 2200, className: ok ? "is-saved" : "is-failed", labelSelector: ".chart-dl-btn-label", @@ -237,7 +236,7 @@ function _chipShareUrl() { async function _copyChipShareLink(btn) { const url = _chipShareUrl(); const ok = await copyToClipboard(url); - flashButtonLabel(btn, ok ? "Copied!" : "Copy failed — select & ⌘C", { + flashButtonLabel(btn, ok ? _i('chipDetail.flashCopied') : _i('chipDetail.flashCopyFailed'), { holdMs: ok ? 1600 : 3500, className: ok ? "is-copied" : "is-copy-failed", labelSelector: ".copy-btn-label", @@ -286,31 +285,30 @@ function renderFingerprintSection(slug, sample) {
- 02 · Performance fingerprint -

How this chip sits across the spectrum

+ ${_i('chipDetail.fingerprintEyebrow')} +

${_i('chipDetail.fingerprintTitle')}

- Each axis is one suite. 100 % is the global best primary metric - for that suite — your chip's normalised score sits inside. + ${_i('chipDetail.fingerprintSub1')} ${missing.length - ? `Suites without a submission collapse to the centre (${missing.map((sid) => SUITE_META[sid]?.letter).filter(Boolean).join(", ")}).` + ? _i('chipDetail.fingerprintSubMissing', `Suites without a submission collapse to the centre (${missing.map((sid) => SUITE_META[sid]?.letter).filter(Boolean).join(", ")}).`) : ""}

-
    +
      ${cells}
@@ -352,7 +350,7 @@ function _mountFingerprintChart(el, slug, sample) { labels, datasets: [ { - label: "Global best", + label: _i('chipDetail.globalBest'), data: reference, borderColor: refColor, borderDash: [4, 4], @@ -442,11 +440,11 @@ function renderScalingSection(slug, sample) { const breakdownRows = data.suites.map((s) => { const bestPerCount = chipCounts.map((c) => { const cell = s.perCount.get(c); - if (!cell || cell.value == null) return ``; + if (!cell || cell.value == null) return ``; const pct = Math.round(cell.normalized * 100); const display = formatPrimary(cell.value, s.sid); return ` - + ${pct}% ${esc(display || "—")} @@ -465,34 +463,29 @@ function renderScalingSection(slug, sample) {
- 03 · Scaling across chip-counts -

Does going wide actually pay off?

+ ${_i('chipDetail.scalingEyebrow')} +

${_i('chipDetail.scalingTitle')}

-

- Bars are normalised to this chip's best result on each suite — - ×N at 100 % means that fan-out wins the suite among this chip's - variants. Chip-counts without a submission for a suite show as - gaps; zoom out via Compare to put another chip on the same axes. -

+

${_i('chipDetail.scalingSub')}

-
    +
      ${legendItems}
-
+
${breakdownRows}
@@ -570,9 +563,9 @@ function _mountScalingChart(el, slug, sample) { const suite = data.suites[sIdx]; const cnt = chipCounts[cIdx]; const cell = suite?.perCount.get(cnt); - if (!cell || cell.value == null) return `×${cnt}: no submission`; + if (!cell || cell.value == null) return `×${cnt}: ${_i('chipDetail.noSubmission')}`; const display = formatPrimary(cell.value, suite.sid); - return `×${cnt}: ${display} (${ctx.parsed.y}% of best)`; + return `×${cnt}: ${display} (${ctx.parsed.y}% ${_i('chipDetail.ofBest')})`; }, }, }, @@ -636,10 +629,10 @@ function renderSimilarChipsSection(slug, latestRid, sectionNum = "04") {
- ${sectionNum} · Peers -

Compare with similar chips

+ ${sectionNum}${_i('chipDetail.peersEyebrow')} +

${_i('chipDetail.peersTitle')}

-

Chips that compete on the same workload suites — sorted by suite overlap, same-vendor first.

+

${_i('chipDetail.peersSub')}

${tiles}
@@ -670,7 +663,7 @@ function renderSuiteCard(sid, row, chipSlug) { ${esc(meta.letter)} ${esc(meta.title)}
-
Not submitted
+
${_i('chipDetail.notSubmitted')}
`; } @@ -702,7 +695,7 @@ function renderSuiteCard(sid, row, chipSlug) { // (more visibly) in a tiny hint footer so the modifier-click path is // findable without a separate help layer. const chipLabel = row._chip_label || "this chip"; - const cardTitle = `Click to open this run · Cmd/Ctrl-click to see all ${chipLabel} runs in Suite ${meta.letter}`; + const cardTitle = _i('chipDetail.cardTitle', `Click to open this run · Cmd/Ctrl-click to see all ${chipLabel} runs in Suite ${meta.letter}`); // Now that chip_count variants share a chip-detail page, the "best // per suite" run can land on any fan-out (×1 vs ×4 vs ×8). Surface @@ -724,7 +717,7 @@ function renderSuiteCard(sid, row, chipSlug) { ${esc(meta.title)} ${rank ? ` + title="${_i('chipDetail.rankedInSuite', `Ranked #${rank.rank} of ${rank.total} in Suite ${esc(meta.letter)}`)}"> #${rank.rank} / ${rank.total} ` : ""} @@ -733,16 +726,16 @@ function renderSuiteCard(sid, row, chipSlug) { ${esc(num)} ${unit ? `${esc(unit)}` : ""} ${showCountBadge - ? `×${bestCount}` + ? `×${bestCount}` : ""}
${fwLine}${row.precision ? ` · ${esc(row.precision)}` : ""}${row.date ? ` · ${esc(fmtDate(row.date))}` : ""}
`; @@ -766,14 +759,14 @@ function renderRunsTable(rs) { - - ${showChipCol ? `` : ""} - - - - - - + + ${showChipCol ? `` : ""} + + + + + + @@ -800,7 +793,7 @@ function renderRunRow(row, showChipCol) { // keydown delegate fires openModal on Enter/Space. Native // semantics stay so screen-reader column headers still pair with // each cell. - const a11yLabel = `Open run details: ${meta ? meta.title + " · " : ""}${row.framework || ""} ${display || ""}`.trim(); + const a11yLabel = `${_i('chipDetail.openRunDetails')} ${meta ? meta.title + " · " : ""}${row.framework || ""} ${display || ""}`.trim(); return ` (window._i ? window._i(k, r) : k); const DATASET_VERSION = "2026.07.08"; const API_BASE = typeof location !== "undefined" ? `${location.origin}${location.pathname}` : ""; @@ -14,14 +15,14 @@ export function render({ el }) { el.innerHTML = `
Reference -

Cite the dataset

-

Three levels of citation — framework, versioned snapshot, and individual reproducible runs. All community results are CC BY 4.0.

+

${_i('cite.title')}

+

${_i('cite.lead')}

-

1 · Project

-

Cite the AccelMark benchmarking framework and methodology.

+

1 · Project

+

${_i('cite.projectDesc')}

${esc(PROJECT_BIBTEX)}
@@ -39,13 +40,13 @@ export function render({ el }) {

3 · Single result

-

Open any result on the Results page → use Copy link, Copy Markdown, or Copy BibTeX in the detail panel. Each run links to result.json, runner hash, and reproduction script.

- Browse results → +

${_i('cite.singleResultDesc')}

+ ${_i('cite.browseResults')}
-

Machine-readable API

+

${_i('cite.apiTitle')}

`; diff --git a/leaderboard/site/assets/js/views/compare.js b/leaderboard/site/assets/js/views/compare.js index 1455260..513d7e3 100644 --- a/leaderboard/site/assets/js/views/compare.js +++ b/leaderboard/site/assets/js/views/compare.js @@ -26,7 +26,7 @@ import { SUITE_ORDER, SUITE_META, SUITE_COLUMNS, formatMetric, rowByRunId, bestRowForRunInSuite, chipCloudData, - representativeRunForChip, + representativeRunForChip, rowsForChip, vendorColor, } from "../data.js"; import { esc, fmtNum, buildHash, chipHref, parseHash, shortVersion, @@ -35,6 +35,7 @@ import { import { basketGet, basketHas, basketToggle, basketOnChange, } from "../router.js"; +const _i = (k, r) => (window._i ? window._i(k, r) : k); export function render({ el, query }) { // Seed from ?runs=a,b,c (back-compat: ?chips=…). Used both for @@ -75,18 +76,17 @@ export function render({ el, query }) { if (runIds.length === 0) { el.innerHTML = `
- Compare -

Side-by-Side Comparison

+ ${_i('compare.eyebrow')} +

${_i('compare.title')}

- Pick platforms below to start a head-to-head across every metric. - You can also tick runs from - any results page - to compare specific framework and precision configurations. + ${_i('compare.sub1')} + ${_i('compare.sub2')} + ${_i('compare.sub3')}

${renderChipCloudBlock({ - title: "Quick-add by platform", - hint: "Each click adds that platform's most recent run. Choose any two or more.", + title: _i('compare.quickadd'), + hint: _i('compare.quickaddHint'), compact: false, })} `; @@ -109,24 +109,24 @@ export function render({ el, query }) { if (seeds.length === 0) { el.innerHTML = `
- Compare -

Side-by-Side Comparison

+ ${_i('compare.eyebrow')} +

${_i('compare.title')}

- This comparison link refers to ${runIds.length === 1 ? "a run" : `${runIds.length} runs`} - that ${runIds.length === 1 ? "is" : "are"} no longer in the dataset - (re-uploaded or pruned). Pick platforms below to start a new comparison. + ${_i('compare.stalePrefix')} ${runIds.length === 1 ? _i('compare.aRun') : `${runIds.length} ${_i('compare.runs')}`} + ${_i('compare.staleMid')} + ${_i('compare.staleSuffix')}

- Comparing - ${runIds.length} stale ${runIds.length === 1 ? "run" : "runs"} + ${_i('compare.comparing')} + ${runIds.length} ${_i('compare.stale')} ${runIds.length === 1 ? _i('compare.run') : _i('compare.runs')}
- +
${renderChipCloudBlock({ - title: "Quick-add by platform", - hint: "Each click adds that platform's most recent run. Choose any two or more.", + title: _i('compare.quickadd'), + hint: _i('compare.quickaddHint'), compact: false, })} `; @@ -172,34 +172,34 @@ export function render({ el, query }) { el.innerHTML = `
- Compare -

${esc(meta.title)}

-

${esc(meta.tagline)}

+ ${_i('compare.eyebrow')} +

${esc(_i('suite.' + suiteId + '.title'))}

+

${esc(_i('suite.' + suiteId + '.tagline'))}

${renderChipCloudBlock({ - title: "Quick-add by platform", - hint: "Click a platform to add or remove its most recent run. Already-selected platforms are highlighted.", + title: _i('compare.quickAddPlatform'), + hint: _i('compare.quickAddPlatformHint'), compact: true, })}
- Comparing + ${_i('compare.comparing')} ${chips.map((c) => renderBasketChip(c)).join("")}
- +
- Suite + ${_i('compare.suite')}
${SUITE_ORDER.map((sid) => renderSuitePill(sid, sid === suiteId)).join("")}
@@ -208,17 +208,17 @@ export function render({ el, query }) { ${suiteEmpty ? `
-

None of the selected configurations have Suite ${esc(meta.letter)} · ${esc(meta.title)} data.

+

${_i('compare.noSuiteData1')} Suite ${esc(meta.letter)} · ${esc(_i('suite.' + suiteId + '.title'))} ${_i('compare.noSuiteData2')}

${suitesWithData.length ? ` -

Try +

${_i('compare.try')} ${suitesWithData.map((sid) => ` Suite ${esc(SUITE_META[sid].letter)} `).join(" · ")} - instead. + ${_i('compare.instead')}

- ` : `

The platforms you picked have no submissions on file.

`} + ` : `

${_i('compare.noSubmissions')}

`}
` : `
@@ -253,24 +253,16 @@ function renderChipCloudBlock({ title, hint, compact }) { if (!chips.length) return ""; const tiles = chips.map((c) => { const inBasket = basketChipNames.has(c.label); - const subL = c.submissions === 1 ? "submission" : "submissions"; - const suiteL = c.suites.length === 1 ? "suite" : "suites"; - const variantPart = c.variants > 1 ? ` · ${c.variants} chip-count variants` : ""; - // Left-click is intercepted by bindClicks for the toggle add/remove - // basket behaviour; the href is the middle-click / Cmd-click / - // copy-link fallback and points at the chip's overview page so it - // matches every other chip-name link on the site. - // - // a11y: tile doubles as a toggle (left-click) and a navigation link - // (modifier-click). We treat the toggle as the primary action for - // assistive tech — `role="button"` + `aria-pressed` mirrors the - // visual "in-basket" state. Modifier-click is documented in the - // tooltip so non-mouse users know about the secondary affordance. - const a11yTitle = `${c.label}: ${c.submissions} ${subL} across ${c.suites.length} ${suiteL}${variantPart}. Click to ${inBasket ? "remove from" : "add to"} compare basket; Cmd / Ctrl-click to open chip overview.`; + const subL = c.submissions === 1 ? _i('compare.submissionUnit', 'submission') : _i('compare.submissionUnitPlural', 'submissions'); + const suiteL = c.suites.length === 1 ? _i('compare.suiteUnit', 'suite') : _i('compare.suiteUnitPlural', 'suites'); + const variantPart = c.variants > 1 ? ` · ${c.variants} ${_i('compare.chipCountVariants', 'chip-count variants')}` : ""; + const vc = vendorColor(c.vendor); + const a11yTitle = `${c.label}: ${c.submissions} ${subL} ${_i('compare.across')} ${c.suites.length} ${suiteL}${variantPart}. ${_i('compare.clickTo')} ${inBasket ? _i('compare.removeFrom') : _i('compare.addTo')} ${_i('compare.compareBasketCtx', 'compare basket; Cmd / Ctrl-click to open chip overview.')}`; return ` -

Head-to-head charts

+

${_i('compare.headToHeadCharts')}

${esc( active.length === chips.length - ? "Each chip overlaid on the same axes." - : `${active.length} of ${chips.length} chips have Suite ${SUITE_META[suiteId].letter} data — others are listed below.` + ? _i('compare.chartsHintAll') + : `${active.length} ${_i('compare.chartsHintOf')} ${chips.length} ${_i('compare.chartsHintChips', `chips have Suite ${SUITE_META[suiteId].letter} data — others are listed below.`)}` )}

`; @@ -384,10 +376,10 @@ function renderCmpCharts(wrap, suiteId, chips) { dlBtn.type = "button"; dlBtn.dataset.chartDl = (spec.title || "chart").toLowerCase() .replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60); - dlBtn.title = "Download this chart as a PNG image"; + dlBtn.title = _i('compare.downloadChartTitle'); dlBtn.innerHTML = ` - PNG + ${_i('compare.png')} `; canvasWrap.appendChild(dlBtn); body.appendChild(canvasWrap); @@ -396,7 +388,7 @@ function renderCmpCharts(wrap, suiteId, chips) { try { _activeCmpCharts.push(new window.Chart(canvas, spec.config)); } catch (e) { - body.innerHTML = `
Chart failed to render.
`; + body.innerHTML = `
${_i('compare.chartFailed')}
`; } } @@ -404,7 +396,7 @@ function renderCmpCharts(wrap, suiteId, chips) { const note = document.createElement("div"); note.className = "cmp-charts-missing"; const names = missing.map((m) => esc(m.label)).join(", "); - note.innerHTML = `No Suite ${esc(SUITE_META[suiteId].letter)} data: ${names}`; + note.innerHTML = `${_i('compare.noSuiteData')} ${esc(SUITE_META[suiteId].letter)}: ${names}`; wrap.appendChild(note); } } @@ -427,11 +419,11 @@ function _cmpLegend(items) { // legend}). Config goes straight to new Chart(canvas, config). const CMP_CHART_RENDERERS = { - suite_A: (chips) => _offlineConcurrencyBars(chips, "Single-chip offline throughput", "tok/s by concurrency"), - suite_F: (chips) => _offlineConcurrencyBars(chips, "Edge offline throughput", "tok/s by concurrency"), - suite_G: (chips) => _offlineConcurrencyBars(chips, "MoE offline throughput", "tok/s by concurrency"), + suite_A: (chips) => _offlineConcurrencyBars(chips, _i('compare.chartSingleChipOffline'), _i('compare.chartTokByConcurrency')), + suite_F: (chips) => _offlineConcurrencyBars(chips, _i('compare.chartEdgeOffline'), _i('compare.chartTokByConcurrency')), + suite_G: (chips) => _offlineConcurrencyBars(chips, _i('compare.chartMoeOffline'), _i('compare.chartTokByConcurrency')), suite_B: (chips) => [ - ..._offlineConcurrencyBars(chips, "Multi-chip total throughput", "tok/s by concurrency"), + ..._offlineConcurrencyBars(chips, _i('compare.chartMultiChipTotal'), _i('compare.chartTokByConcurrency')), ..._perChipThroughputBars(chips), ], suite_C: (chips) => _quantThroughputBars(chips), @@ -459,7 +451,7 @@ function _offlineConcurrencyBars(chips, title, subtitle) { const data = chips.map((c) => c.suiteRow.offline_throughput ?? null); return [{ title, - subtitle: "Peak offline throughput", + subtitle: _i('compare.chartPeakOffline'), height: 200, config: { type: "bar", @@ -477,7 +469,7 @@ function _offlineConcurrencyBars(chips, title, subtitle) { plugins: { legend: { display: false } }, scales: { x: { ticks: { color: C.text, font: { size: 11 } }, grid: { color: C.grid } }, - y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString() }, grid: { color: C.grid }, title: { display: true, text: "tokens / sec", color: C.text, font: { size: 11 } } }, + y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString() }, grid: { color: C.grid }, title: { display: true, text: _i('compare.chartTokensPerSec'), color: C.text, font: { size: 11 } } }, }, }, }, @@ -516,8 +508,8 @@ function _offlineConcurrencyBars(chips, title, subtitle) { responsive: true, maintainAspectRatio: false, plugins: { legend: { display: false } }, scales: { - x: { ticks: { color: C.text, font: { size: 11 } }, grid: { color: C.grid }, title: { display: true, text: "concurrency", color: C.text, font: { size: 11 } } }, - y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString() }, grid: { color: C.grid }, title: { display: true, text: "tokens / sec", color: C.text, font: { size: 11 } } }, + x: { ticks: { color: C.text, font: { size: 11 } }, grid: { color: C.grid }, title: { display: true, text: _i('compare.chartConcurrency'), color: C.text, font: { size: 11 } } }, + y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString() }, grid: { color: C.grid }, title: { display: true, text: _i('compare.chartTokensPerSec'), color: C.text, font: { size: 11 } } }, }, }, }, @@ -531,8 +523,8 @@ function _perChipThroughputBars(chips) { const data = chips.map((c) => c.suiteRow.tokens_per_sec_per_chip ?? null); if (data.every((v) => v == null)) return []; return [{ - title: "Per-chip throughput", - subtitle: "tok/s per accelerator, scales with hardware count", + title: _i('compare.chartPerChipThroughput'), + subtitle: _i('compare.chartPerChipSub'), height: 200, config: { type: "bar", @@ -550,7 +542,7 @@ function _perChipThroughputBars(chips) { plugins: { legend: { display: false } }, scales: { x: { ticks: { color: C.text, font: { size: 11 } }, grid: { color: C.grid } }, - y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString() }, grid: { color: C.grid }, title: { display: true, text: "tok / sec / chip", color: C.text, font: { size: 11 } } }, + y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString() }, grid: { color: C.grid }, title: { display: true, text: _i('compare.chartTokPerSecPerChip'), color: C.text, font: { size: 11 } } }, }, }, }, @@ -598,8 +590,8 @@ function _quantThroughputBars(chips) { }; }); return [{ - title: "Throughput across quantization formats", - subtitle: "tok/s by precision · grouped by chip", + title: _i('compare.chartQuantThroughput'), + subtitle: _i('compare.chartQuantSub'), height: 230, config: { type: "bar", @@ -608,8 +600,8 @@ function _quantThroughputBars(chips) { responsive: true, maintainAspectRatio: false, plugins: { legend: { display: false } }, scales: { - x: { ticks: { color: C.text, font: { size: 11 } }, grid: { color: C.grid }, title: { display: true, text: "precision", color: C.text, font: { size: 11 } } }, - y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString(undefined, { maximumFractionDigits: 0 }) }, grid: { color: C.grid }, title: { display: true, text: "tokens / sec", color: C.text, font: { size: 11 } } }, + x: { ticks: { color: C.text, font: { size: 11 } }, grid: { color: C.grid }, title: { display: true, text: _i('compare.chartPrecision'), color: C.text, font: { size: 11 } } }, + y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString(undefined, { maximumFractionDigits: 0 }) }, grid: { color: C.grid }, title: { display: true, text: _i('compare.chartTokensPerSec'), color: C.text, font: { size: 11 } } }, }, }, }, @@ -620,12 +612,12 @@ function _quantThroughputBars(chips) { function _longContextLatency(chips) { const C = _cmpChartColors(); const buckets = [ - { key: "ttft_p50", label: "TTFT p50" }, - { key: "ttft_p90", label: "TTFT p90" }, - { key: "ttft_p99", label: "TTFT p99" }, - { key: "tpot_p50", label: "TPOT p50" }, - { key: "tpot_p90", label: "TPOT p90" }, - { key: "tpot_p99", label: "TPOT p99" }, + { key: "ttft_p50", label: _i('compare.chartTtftP50') }, + { key: "ttft_p90", label: _i('compare.chartTtftP90') }, + { key: "ttft_p99", label: _i('compare.chartTtftP99') }, + { key: "tpot_p50", label: _i('compare.chartTpotP50') }, + { key: "tpot_p90", label: _i('compare.chartTpotP90') }, + { key: "tpot_p99", label: _i('compare.chartTpotP99') }, ]; const datasets = chips.map((c, i) => { const v = (c.suiteRow.viz && c.suiteRow.viz.interactive) || {}; @@ -641,8 +633,8 @@ function _longContextLatency(chips) { const filtered = datasets.filter((ds) => ds.data.some((v) => v != null)); if (!filtered.length) return []; return [{ - title: "Long-context latency", - subtitle: "ms · TTFT (prefill) and TPOT (decode) percentiles", + title: _i('compare.chartLongContextLatency'), + subtitle: _i('compare.chartLongContextSub'), height: 280, config: { type: "bar", @@ -652,7 +644,7 @@ function _longContextLatency(chips) { responsive: true, maintainAspectRatio: false, plugins: { legend: { display: false } }, scales: { - x: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v >= 1000 ? (v / 1000).toFixed(1) + "s" : v + "ms" }, grid: { color: C.grid }, title: { display: true, text: "latency", color: C.text, font: { size: 11 } } }, + x: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v >= 1000 ? (v / 1000).toFixed(1) + "s" : v + "ms" }, grid: { color: C.grid }, title: { display: true, text: _i('compare.chartLatency'), color: C.text, font: { size: 11 } } }, y: { ticks: { color: C.text, font: { size: 11 } }, grid: { color: C.grid } }, }, }, @@ -715,8 +707,8 @@ function _scalingCurves(chips) { }); return [ { - title: "Throughput by chip count", - subtitle: "tok/s · one line per chip", + title: _i('compare.chartThroughputByCount'), + subtitle: _i('compare.chartThroughputByCountSub'), height: 240, config: { type: "line", @@ -726,15 +718,15 @@ function _scalingCurves(chips) { plugins: { legend: { display: false } }, scales: { x: { ticks: { color: C.text, font: { size: 11 } }, grid: { color: C.grid } }, - y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString() }, grid: { color: C.grid }, title: { display: true, text: "tokens / sec", color: C.text, font: { size: 11 } } }, + y: { ticks: { color: C.text, font: { size: 11 }, callback: (v) => v.toLocaleString() }, grid: { color: C.grid }, title: { display: true, text: _i('compare.chartTokensPerSec'), color: C.text, font: { size: 11 } } }, }, }, }, legend: chips.map((c, i) => ({ color: _palette(i), label: c.label })), }, { - title: "Scaling efficiency vs linear ideal", - subtitle: "% · 100 % is perfect linear scaling", + title: _i('compare.chartScalingEfficiency'), + subtitle: _i('compare.chartScalingEfficiencySub'), height: 240, config: { type: "line", @@ -742,9 +734,9 @@ function _scalingCurves(chips) { labels, datasets: [ ...effDs, - { label: "Linear ideal", data: counts.map(() => 100), + { label: _i('compare.chartLinearIdeal'), data: counts.map(() => 100), borderColor: C.text, borderDash: [6, 4], pointRadius: 0, fill: false, tension: 0 }, - { label: "Good (80 %)", data: counts.map(() => 80), + { label: _i('compare.chartGood80'), data: counts.map(() => 80), borderColor: "#2dd4bf", borderDash: [3, 3], pointRadius: 0, fill: false, tension: 0 }, ], }, @@ -753,12 +745,12 @@ function _scalingCurves(chips) { plugins: { legend: { display: false } }, scales: { x: { ticks: { color: C.text, font: { size: 11 } }, grid: { color: C.grid } }, - y: { min: 0, max: 110, ticks: { color: C.text, font: { size: 11 }, callback: (v) => v + "%" }, grid: { color: C.grid }, title: { display: true, text: "efficiency %", color: C.text, font: { size: 11 } } }, + y: { min: 0, max: 110, ticks: { color: C.text, font: { size: 11 }, callback: (v) => v + "%" }, grid: { color: C.grid }, title: { display: true, text: _i('compare.chartEfficiencyPct'), color: C.text, font: { size: 11 } } }, }, }, }, legend: chips.map((c, i) => ({ color: _palette(i), label: c.label })) - .concat([{ color: "#2dd4bf", label: "Good (80 %)" }]), + .concat([{ color: "#2dd4bf", label: _i('compare.chartGood80') }]), }, ]; } @@ -777,6 +769,22 @@ function renderSuitePill(sid, active) { `; } +// Returns all unique framework+precision variants available for a chip. +// Used by renderBasketChip to show a dropdown when the chip has 2+ variants +// so users can compare runs on the same framework, not vLLM vs SGLang. +function getSiblingRuns(rid) { + const row = rowByRunId(rid); + if (!row) return []; + const all = rowsForChip(row._chip_slug); + const seen = new Set(); + return all.filter(r => { + const key = `${r.framework}|${r.framework_version}|${r.precision}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + // Basket strip pill — chip name on top line, framework + version + precision // on the second so the user can tell two same-chip runs apart at a glance. // Plain click on the name opens the seed run's detail modal; Cmd-click @@ -786,16 +794,30 @@ function renderBasketChip(c) { const ver = shortVersion(r.framework_version); const fwLine = [r.framework, ver].filter(Boolean).join(" "); const detail = [fwLine, r.precision].filter(Boolean).join(" · "); - // Chip-name link goes to the chip overview (every chip name on the - // site should navigate to /chip/). The × button still removes - // the specific run from the basket — that's the basket-local action. + + // Build framework selector if this chip has multiple variants + const siblings = getSiblingRuns(c.rid); + let metaHtml; + if (siblings.length > 1) { + const opts = siblings.map(s => { + const sv = shortVersion(s.framework_version); + const sfw = [s.framework, sv].filter(Boolean).join(" "); + const sdetail = [sfw, s.precision].filter(Boolean).join(" · "); + const sid = s.run_id || s.submission; + return ``; + }).join(""); + metaHtml = ``; + } else { + metaHtml = detail ? `${esc(detail)}` : ""; + } + return `
${esc(c.label)} - ${detail ? `${esc(detail)}` : ""} + ${metaHtml}
- + ${chips.map((c) => renderChipHead(c, meta)).join("")} @@ -823,7 +845,7 @@ function renderCmpTable(suiteId, cols, chips) { function renderChipHead(c, meta) { const r = c.suiteRow; - let metaLine = `No Suite ${esc(meta.letter)} data`; + let metaLine = `${_i('compare.noSuiteData')} ${esc(meta.letter)}`; if (r) { const fw = r.framework || ""; const ver = shortVersion(r.framework_version); @@ -881,7 +903,7 @@ function renderCmpRow(col, chips) { return max > 0 ? v / max : 0; }; - const dirLabel = col.direction === "asc" ? "lower is better" : "higher is better"; + const dirLabel = col.direction === "asc" ? _i('compare.lowerIsBetter') : _i('compare.higherIsBetter'); return ` @@ -939,7 +961,7 @@ function _shareUrlForBasket(suiteId) { async function _copyShareLink(btn, suiteId) { const url = _shareUrlForBasket(suiteId); const ok = await copyToClipboard(url); - flashButtonLabel(btn, ok ? "Copied!" : "Copy failed — select & ⌘C", { + flashButtonLabel(btn, ok ? _i('compare.flashCopied') : _i('compare.flashCopyFailed'), { holdMs: ok ? 1600 : 3500, className: ok ? "is-copied" : "is-copy-failed", labelSelector: ".copy-btn-label", @@ -1022,7 +1044,9 @@ function bindClicks(el) { ev.preventDefault(); const sid = suitePill.dataset.suite; if (sid && sid !== suiteId) { + const sy = window.scrollY; location.hash = buildHash("/compare", { suite: sid }); + requestAnimationFrame(() => requestAnimationFrame(() => window.scrollTo(0, sy))); } return; } @@ -1036,20 +1060,33 @@ function bindClicks(el) { return; } }); + + // Framework/precision selector — swap a chip's run variant in-place + el.addEventListener("change", (ev) => { + if (!location.hash.startsWith("#/compare")) return; + const sel = ev.target.closest(".cmp-basket-chip-select"); + if (!sel) return; + const oldRid = sel.dataset.rid; + const newRid = sel.value; + if (oldRid && newRid && oldRid !== newRid) { + basketToggle(oldRid); + basketToggle(newRid); + } + }); } async function _downloadCmpChart(btn, suiteId) { const wrap = btn.closest(".cmp-chart-canvas"); const canvas = wrap && wrap.querySelector("canvas"); if (!canvas) { - flashButtonLabel(btn, "Failed", { holdMs: 2000, className: "is-failed", labelSelector: ".chart-dl-btn-label" }); + flashButtonLabel(btn, _i('compare.flashFailed'), { holdMs: 2000, className: "is-failed", labelSelector: ".chart-dl-btn-label" }); return; } const sectionSlug = btn.dataset.chartDl || "chart"; const ok = await downloadCanvasAsPng(canvas, { filename: `compare-${suiteId.replace(/^suite_/, "suite-")}-${sectionSlug}.png`, }); - flashButtonLabel(btn, ok ? "Saved" : "Failed", { + flashButtonLabel(btn, ok ? _i('compare.flashSaved') : _i('compare.flashFailed'), { holdMs: ok ? 1400 : 2200, className: ok ? "is-saved" : "is-failed", labelSelector: ".chart-dl-btn-label", diff --git a/leaderboard/site/assets/js/views/contributor.js b/leaderboard/site/assets/js/views/contributor.js index 14e64ae..0fe435f 100644 --- a/leaderboard/site/assets/js/views/contributor.js +++ b/leaderboard/site/assets/js/views/contributor.js @@ -3,6 +3,7 @@ import { esc, fmtNum, fmtDate } from "../utils.js"; import { contributorByHandle, BADGE_DEFS, contributorIndex } from "../contributors.js"; import { SUITE_META } from "../data.js"; +const _i = (k, r) => (window._i ? window._i(k, r) : k); function badgeHtml(ids) { return (ids || []).map((id) => { @@ -25,9 +26,9 @@ export function render({ params, el }) { el.innerHTML = `
No published results for @${esc(handle)}.
- Make sure your GitHub login matches submitted_by in the PR.
+ ${_i('contributor.githubHint')} submitted_by in the PR.
← All contributors - Submit a result + ${_i('contributor.submitResult')}
`; return; @@ -61,8 +62,8 @@ export function render({ params, el }) { ${c.firstChips.length ? `
-

First results

- First published benchmark on the leaderboard for this hardware +

${_i('contributor.firstResults')}

+ ${_i('contributor.firstBenchmark')}
${c.firstChips.map((chip) => ` @@ -74,7 +75,7 @@ export function render({ params, el }) {
-

Published runs

+

${_i('contributor.publishedRuns')}

Latest activity: ${esc(fmtDate(c.latestDate))}
diff --git a/leaderboard/site/assets/js/views/contributors.js b/leaderboard/site/assets/js/views/contributors.js index fc1c9fe..3263ed7 100644 --- a/leaderboard/site/assets/js/views/contributors.js +++ b/leaderboard/site/assets/js/views/contributors.js @@ -1,16 +1,18 @@ // contributors.js (view) — public contributor leaderboard. - import { esc, fmtNum } from "../utils.js"; import { contributorIndex, BADGE_DEFS } from "../contributors.js"; import { summary } from "../data.js"; import { communityHeader } from "../community-nav.js"; +const _i = (k, r) => (window._i ? window._i(k, r) : k); function badgeHtml(ids) { if (!ids?.length) return ""; return ids.map((id) => { const b = BADGE_DEFS[id]; if (!b) return ""; - return `${esc(b.label)}`; + const labelKey = 'badge.' + id + '.label'; + const descKey = 'badge.' + id + '.desc'; + return `${esc(_i(labelKey, b.label))}`; }).join(""); } @@ -21,28 +23,27 @@ export function render({ el }) { el.innerHTML = ` ${communityHeader( "contributors", - "Contributor index", - `${fmtNum(list.length)} contributors · ${fmtNum(s.total)} published runs · ${fmtNum(s.verified)} verified. ` + - `Profiles are derived from submitted_by in merged results (GitHub handle); no separate registration.` + _i('contributors.title'), + `${fmtNum(list.length)} ${_i('contributors.count')}` )}
-

Ranking

- Score = runs + verified×3 + platforms×4 + first-result×12 + runner×5 +

${_i('contributors.ranking')}

+ ${_i('contributors.scoreDesc')}
${list.length ? `
SuiteChipsFrameworkPrecisionPrimary metricDateSubmitterTier${_i('chipDetail.thSuite')}${_i('chipDetail.thChips')}${_i('chipDetail.thFramework')}${_i('chipDetail.thPrecision')}${_i('chipDetail.thPrimaryMetric')}${_i('chipDetail.thDate')}${_i('chipDetail.thSubmitter')}${_i('chipDetail.thTier')}
Metric${_i('compare.metric')}
- - - - - - - + + + + + + + @@ -63,15 +64,16 @@ export function render({ el }) {
RankContributorRunsVerifiedPlatformsAttributionsScore${_i('contributors.rank')}${_i('contributors.contributor')}${_i('contributors.runs')}${_i('contributors.verified')}${_i('contributors.platforms')}${_i('contributors.attribs')}${_i('contributors.score')}
- ` : `

No contributors yet — submit the first result.

`} + ` : `

${_i('contributors.empty')} — ${_i('contributors.submitFirst')}.

`}
diff --git a/leaderboard/site/assets/js/views/home.js b/leaderboard/site/assets/js/views/home.js index 11fcf40..f1022f9 100644 --- a/leaderboard/site/assets/js/views/home.js +++ b/leaderboard/site/assets/js/views/home.js @@ -16,6 +16,7 @@ import { rowsForSuite, suiteFacts, chipCloudData, summary, recent, recentSince, formatPrimary, suiteChartAxisLabel, suiteChartBlurb, suiteChartHead, suiteChartPurpose, + implOpsLabel, } from "../data.js"; import { contributorIndex } from "../contributors.js"; import { @@ -23,6 +24,7 @@ import { shortVersion, shortModel, submitterHandle, } from "../utils.js"; +const _i = (k,r) => (window._i ? window._i(k,r) : k); const TOP_N = 8; const RECENT_WINDOW_DAYS = 7; @@ -107,9 +109,9 @@ function renderDistChart() { var gxMn=valid.map(function(d){return getV(d);}).filter(function(v){return v>0;}); gxMn=gxMn.length?Math.max(0.5,Math.min.apply(null,gxMn)*0.5):1; suites.forEach(function(suite){var sd=valid.filter(function(d){return d.suite===suite;});if(!sd.length)return;var inner=document.createElement('div');inner.style.cssText='background:var(--bg-elev);border:1px solid var(--border-soft);border-radius:var(--r-md);padding:6px';inner.innerHTML='
'+esc(suiteChartHead(suite))+'
'+esc(suiteChartPurpose(suite))+' ('+sd.length+')
';el.appendChild(inner);setTimeout(function(){var el2=inner.querySelector('.sm-chart');if(!el2)return;var c=echarts.init(el2);var vends=[...new Set(sd.map(function(d){return d.chip_vendor;}))];var ser=vends.map(function(v){return{name:v,type:'scatter',data:sd.filter(function(d){return d.chip_vendor===v&&getV(d)>0;}).map(function(d){return{value:[getV(d),(d.scenarios&&d.scenarios.online&&d.scenarios.online.is_valid&&d.scenarios.online.throughput>0)?d.scenarios.online.throughput:0.05],submission:d};}),symbol:'circle',symbolSize:6,itemStyle:{color:distVc(v),opacity:.7},emphasis:{scale:1.5}};});c.setOption({tooltip:{trigger:'item',backgroundColor:TBG,borderColor:TBR,textStyle:{color:TFG,fontSize:9},formatter:function(p){return distTip(distSubOf(p),mt,getV);}},grid:{left:36,right:6,top:6,bottom:18},xAxis:{type:'log',min:gxMn,axisLabel:{fontSize:6,color:AT},splitLine:{show:false}},yAxis:{type:'log',axisLabel:{fontSize:6,color:AT},splitLine:{show:false},min:0.05},series:ser});distBindClick(c);},30);}); - if(info)info.textContent='Each suite in its own chart · '+valid.length+' recipes'; + if(info)info.textContent=_i('dist.info.smallchart')+' · '+valid.length+' recipes'; var cap0=document.getElementById('home-dist-caption'); - if(cap0)cap0.textContent='Figure 1. Per-suite mini scatter plots (throughput × QPS); each point is one serving recipe.'; + if(cap0)cap0.textContent=_i('dist.caption.small'); return; } @@ -155,16 +157,15 @@ function renderDistChart() { chart.resize(); var normHint=distNormalize&&(distView!=='beeswarm'&&distView!=='byframework')?' · Normalize applies to column views only':''; if(distNormalize&&(distView==='beeswarm'||distView==='byframework'))normHint=' · normalized to % of column best'; - if(info)info.innerHTML='Showing '+valid.length+' recipes across '+suites.length+' suites · '+esc(mt.label)+''+esc(normHint); + if(info)info.innerHTML=_i('dist.info.showing',{n:valid.length,s:suites.length,m:esc(mt.label)})+esc(normHint); var cap=document.getElementById('home-dist-caption'); if(cap){ - var base='Figure 1. Each point is one serving recipe (framework × precision × hardware); larger markers denote best-in-class per vendor.'; - if(distNormalize&&(distView==='beeswarm'||distView==='byframework'))cap.textContent=base+' Y-axis shows % of column best (linear scale).'; - else if(distView==='beeswarm'||distView==='byframework')cap.textContent=base+' Y-axis log-scaled.'; - else if(distView==='scatter')cap.textContent='Figure 1. Throughput (x) vs online QPS (y); each point is one serving recipe. Both axes log-scaled.'; - else if(distView==='density')cap.textContent='Figure 1. Kernel-density-style overlap of recipe throughput; darker regions indicate more submissions.'; - else if(distView==='heatmap')cap.textContent='Figure 1. Best throughput per chip × suite cell; color intensity encodes '+mt.label.toLowerCase()+'.'; - else cap.textContent='Figure 1. Per-suite mini scatter plots (throughput × QPS); each point is one serving recipe.'; + if(distNormalize&&(distView==='beeswarm'||distView==='byframework'))cap.textContent=_i('dist.caption.beeswarm_norm'); + else if(distView==='beeswarm'||distView==='byframework')cap.textContent=_i('dist.caption.beeswarm'); + else if(distView==='scatter')cap.textContent=_i('dist.caption.scatter'); + else if(distView==='density')cap.textContent=_i('dist.caption.density'); + else if(distView==='heatmap')cap.textContent=_i('dist.caption.heatmap'); + else cap.textContent=_i('dist.caption.small'); } } @@ -179,21 +180,21 @@ function renderCommunityHub() { ${fmtNum(c.verified)} ${fmtNum(c.score)} - `).join("") : `No contributors yet.`; + `).join("") : `${_i('community.empty')}`; return `
- 05 · Community -

Contribution index

+ ${_i('community.eyebrow')} +

${_i('community.title')}

- All contributors → + ${_i('community.all')}
-

Ranked by verified runs and reproducible evidence — each submission links to full artifacts.

+

${_i('community.subtitle')}

- + ${rows}
RankContributorRunsVerifiedScore
${_i('community.rank')}${_i('community.contributor')}${_i('community.runs')}${_i('community.verified')}${_i('community.score')}
@@ -201,6 +202,76 @@ function renderCommunityHub() { `; } +// Rankings-reverse hero chart — A100 vs 5090 across Suite A (bandwidth-bound), +// Suite D (compute-bound), and MSRP. Pure inline SVG themed via CSS variables. +// Unified perspective from A100: green = A100 advantage, red = A100 disadvantage +function renderHeroChart() { + const C = { + blue: "var(--accent-2, #3b82f6)", + warm: "#e8904f", + }; + const W = 126, H = 140, PAD = { t: 20, r: 6, b: 26, l: 40 }; + const PW = W - PAD.l - PAD.r, PH = H - PAD.t - PAD.b; + + function panel(data, yFmt) { + const max = Math.max(...data.map(d => d.v)) * 1.12; + const y = v => PAD.t + PH - (v / max) * (PH - 4); + const gap = 22, barW = (PW - gap) / 2; + const xOffset = PAD.l; + + let bars = ""; + data.forEach((d, i) => { + const bx = xOffset + i * (barW + gap); + const by = y(d.v), bh = PAD.t + PH - by; + bars += ``; + bars += `${yFmt(d.v)}`; + bars += `${d.label}`; + }); + + let ticks = ""; + for (let i = 0; i <= 3; i++) { + const v = (max / 3) * i; + const ty = y(v); + ticks += ``; + ticks += `${yFmt(v)}`; + } + + return `${ticks}${bars}`; + } + + const fmtA = v => v >= 1000 ? (v / 1000).toFixed(1) + "k" : v.toFixed(0); + const fmtD = v => v.toFixed(1); + const fmtUSD = v => v >= 1000 ? "$" + (v / 1000).toFixed(0) + "k" : "$" + v; + + const pA = panel([ + { label: "A100", v: 2701, color: C.blue }, + { label: "5090", v: 3487, color: C.warm }, + ], fmtA); + const pD = panel([ + { label: "A100", v: 70.2, color: C.blue }, + { label: "5090", v: 54.3, color: C.warm }, + ], fmtD); + const pP = panel([ + { label: "A100", v: 15000, color: C.blue }, + { label: "5090", v: 1999, color: C.warm }, + ], fmtUSD); + + return `
+
+ Suite A · tok/s${pA} + A100 -29% +
+
+ Suite D · tok/s${pD} + A100 +29% +
+
+ MSRP${pP} + A100 7.5× pricier +
+
`; +} + export function render({ el }) { const s = summary(); // 7-day momentum stat for the hero strip + standalone activity @@ -212,55 +283,60 @@ export function render({ el }) { : ""; el.innerHTML = `
-

- Run your accelerator. - Publish reproducible LLM inference results. -

-

Compare NVIDIA, AMD, Ascend, Apple Silicon, TPU and emerging chips under the same workload suites.

-

- Every row links to result.json, env_info.json, runner hash, - accuracy receipt, and reproduction instructions — open evidence, not just a number on a chart. -

+
+
+

+ ${_i('hero.title1')} + ${_i('hero.title2')} +

+

${_i('hero.subtitle')}

+
+
+ ${_i('hero.chartEyebrow')} + ${renderHeroChart()} +

${_i('hero.chartCaption')}

+
+
-
${fmtNum(s.total)}benchmarks
-
${fmtNum(s.chips)}platforms
-
${fmtNum(s.vendors)}vendors
-
${fmtNum(s.suites)}workloads
-
${fmtNum(s.verified)}verified
+
${fmtNum(s.total)}${_i('hero.kpi.benchmarks')}
+
${fmtNum(s.chips)}${_i('hero.kpi.gpus')}
+
${fmtNum(s.vendors)}${_i('hero.kpi.vendors')}
+
${fmtNum(s.suites)}${_i('hero.kpi.workloads')}
+
${fmtNum(s.verified)}${_i('hero.kpi.verified')}
${recentCount > 0 ? `
${fmtNum(recentCount)} - this week + ${_i('hero.kpi.thisWeek')}
` : ""}
${recentRibbon}
- Why submit -

Get a shareable, citable public record

+ ${_i('why.eyebrow')} +

${_i('why.title')}

-

Reproducible evidence

-

Your run ships with environment fingerprint, runner source hash, and validation receipts — others can rerun and verify, not just trust a screenshot.

+

${_i('why.card1.title')}

+

${_i('why.card1.desc')}

-

Citable submissions

-

Papers and reports can cite project-level, snapshot-level, or per-run BibTeX — with deep links back to your submission artifacts.

+

${_i('why.card3.title')}

+

${_i('why.card3.desc')}

@@ -268,31 +344,31 @@ export function render({ el }) {
- 01 · Explore -

The recipe landscape

+ ${_i('dist.eyebrow')} +

${_i('dist.title')}

- Each dot is a serving recipe — framework, precision, and hardware together. Wider spread within a suite means more room for tuning. + ${_i('dist.subtitle')}
-
-
-
-
-
- +
+
+
+
+
+
Beeswarm - By Framework + ${_i('dist.tab.byframework')} Scatter - - Beeswarm — Grouped by suite; each dot is a recipe. Larger dots = best per vendor.
By Framework — Same layout, grouped by serving framework.
Scatter — Throughput × QPS. Top-right = both strong.
Density — Overlap shading. Darker = more recipes.
Heatmap — Suite × Chip matrix. Color = throughput.
By Suite — One mini-chart per suite. Click a dot to open that platform.
Normalize — Column views only: Y-axis becomes % of column best (0–100%).
+ + ${_i('dist.info.all')}
- Metric + ${_i('dist.metric.label')}
- More views + ${_i('dist.moreviews')}
Density @@ -301,16 +377,16 @@ export function render({ el }) {
-

Figure 1. Each point is one serving recipe (framework × precision × hardware); larger markers denote best-in-class per vendor. Y-axis log-scaled.

+

${_i('home.figure1')} (framework × precision × hardware); larger markers denote best-in-class per vendor. Y-axis log-scaled.

- 02 · Workloads -

Browse by benchmark suite

+ ${_i('workloads.eyebrow')} +

${_i('workloads.title')}

- Each suite tests a fixed model under a specific protocol — from single-GPU throughput to long-context serving. + ${_i('workloads.subtitle')}
@@ -318,10 +394,10 @@ export function render({ el }) {
- 03 · Coverage -

Platform coverage

+ ${_i('coverage.eyebrow')} +

${_i('coverage.title')}

- Tile size reflects submission count. Colour indicates vendor family. + ${_i('coverage.subtitle')}
@@ -330,10 +406,10 @@ export function render({ el }) {
- 04 · Latest activity -

Recent submissions

+ ${_i('recent.eyebrow')} +

${_i('recent.title')}

- See all → + ${_i('recent.seeall')}
@@ -342,24 +418,22 @@ export function render({ el }) {
- 06 · Contribute -

Ready to publish your benchmark?

-

- If you already have the hardware and a supported serving stack, you can go from zero to a merged PR in about ten minutes. -

+ ${_i('contribute.eyebrow')} +

${_i('contribute.title')}

+

${_i('contribute.body')}

    -
  1. Open the submit wizard — pick workload suite, platform, and framework; it prints the exact command and output folder layout.
  2. -
  3. Run the benchmark — one command produces result.json, env_info.json, and accuracy receipts under results/community/.
  4. -
  5. Open a pull request — add that folder; CI checks artifacts. After merge you get a permanent link and citation exports.
  6. +
  7. ${_i('contribute.step1')}
  8. +
  9. ${_i('contribute.step2')}
  10. +
  11. ${_i('contribute.step3')}

- First time with the harness, accuracy rules, or hardware setup? - Read the full contributor guide ↗ + ${_i('contribute.foot')} + ${_i('contribute.guide')}

@@ -386,7 +460,7 @@ export function render({ el }) { ['f-suite','f-vendor','f-framework','f-precision','f-chip'].forEach(function(id){ var field = id === 'f-suite' ? 'suite' : id === 'f-vendor' ? 'chip_vendor' : id === 'f-framework' ? 'framework' : id === 'f-precision' ? 'precision' : 'chip'; var sel = el.querySelector('#'+id); if (!sel) return; - sel.innerHTML = '' + uniq(field).map(function(v){return '';}).join(''); + sel.innerHTML = '' + uniq(field).map(function(v){return '';}).join(''); }); } @@ -468,11 +542,11 @@ function renderSuiteCard(suiteId) {
${esc(meta.letter)} - ${esc(meta.title)} + ${esc(_i('suite.' + suiteId + '.title'))}
${esc(meta.primary.label)}
-

${esc(meta.tagline)}

+

${esc(_i('suite.' + suiteId + '.tagline'))}

${metaLine}
`; @@ -480,8 +554,8 @@ function renderSuiteCard(suiteId) { if (empty) { card.innerHTML = ` ${header} -
Awaiting first submission.
- +
${_i('workloads.awaiting')}
+ `; return card; } @@ -490,7 +564,7 @@ function renderSuiteCard(suiteId) { card.innerHTML = ` ${header}
${body}
- + `; return card; } @@ -508,10 +582,10 @@ function renderSuiteMeta(suiteId, facts) { items.push(`${esc(wl.inputTokens)} → ${esc(wl.outputTokens)} tok`); } if (facts.submissions) { - items.push(`${fmtNum(facts.submissions)} results`); + items.push(`${fmtNum(facts.submissions)} ${_i('home.resultsLabel')}`); } if (facts.chips) { - items.push(`${fmtNum(facts.chips)} chips`); + items.push(`${fmtNum(facts.chips)} ${_i('home.chipsLabel')}`); } return items.join(""); } @@ -529,8 +603,9 @@ function renderLbRow(row, suiteId, rank) { const runId = row.run_id || row.submission || ""; const ver = shortVersion(row.framework_version); const fw = row.framework || ""; + const ops = implOpsLabel(row); const chipRecipe = fw - ? `${esc(row._chip_label)} · ${esc(fw)}${ver ? ` ${esc(ver)}` : ""}` + ? `${esc(row._chip_label)} · ${esc(fw)}${ver ? ` ${esc(ver)}` : ""}${ops ? ` ${esc(ops)}` : ""}` : esc(row._chip_label); const a11yLabel = `Open run details for ${row._chip_label}${fw ? " on " + fw : ""}`; return ` @@ -541,9 +616,8 @@ function renderLbRow(row, suiteId, rank) { data-open-run="${esc(runId)}"> ${rank} - + ${chipRecipe} - ${esc(row._chip_label)}${esc(fw)} ${esc(ver)} ${renderRecipeSub(row)} @@ -609,8 +683,8 @@ function renderChipCloud(container, legendEl) { } const vendors = Array.from(byVendor.values()).sort((a, b) => b.submissions - a.submissions); legendEl.innerHTML = vendors.map((v) => { - const chipsLbl = v.chips === 1 ? "chip" : "chips"; - const subsLbl = v.submissions === 1 ? "result" : "results"; + const chipsLbl = v.chips === 1 ? _i('home.chipLabel') : _i('home.chipsLabel'); + const subsLbl = v.submissions === 1 ? _i('home.resultLabel') : _i('home.resultsLabel'); return ` @@ -631,8 +705,9 @@ function renderRecentRow(row) { const handle = submitterHandle(row.submitted_by); const ver = shortVersion(row.framework_version); const fw = row.framework || ""; + const ops = implOpsLabel(row); const chipRecipe = fw - ? `${esc(row._chip_label)} · ${esc(fw)}${ver ? ` ${esc(ver)}` : ""}` + ? `${esc(row._chip_label)} · ${esc(fw)}${ver ? ` ${esc(ver)}` : ""}${ops ? ` ${esc(ops)}` : ""}` : esc(row._chip_label); // Mirrors renderLbRow: outer
= run-modal trigger, inner chip-name // escapes via modal.js nested-anchor rule to navigate to /chip/. diff --git a/leaderboard/site/assets/js/views/rankings.js b/leaderboard/site/assets/js/views/rankings.js index 4ae3d3b..ed97777 100644 --- a/leaderboard/site/assets/js/views/rankings.js +++ b/leaderboard/site/assets/js/views/rankings.js @@ -27,8 +27,9 @@ import { SUITE_ORDER, SUITE_META, VENDOR_ORDER, SUITE_COLUMNS, formatMetric, - rowsForSuite, + rowsForSuite, implOpsLabel, } from "../data.js"; +const _i = (k, r) => (window._i ? window._i(k, r) : k); import { esc, fmtNum, fmtDate, chipHref, buildHash, parseHash, shortVersion, @@ -119,9 +120,9 @@ export function render({ el, query }) { el.innerHTML = `
- Results · Suite ${esc(meta.letter)} -

${esc(meta.title)}

-

${esc(meta.tagline)} Compare configurations across frameworks and hardware.

+ ${_i('rankings.eyebrow',{letter:esc(meta.letter)})} +

${esc(_i('suite.' + suiteId + '.title', meta.title))}

+

${esc(_i('suite.' + suiteId + '.tagline', meta.tagline))} ${_i('rankings.subtitle')}

@@ -130,21 +131,21 @@ export function render({ el, query }) {
${fmtNum(sorted.length)} - of ${fmtNum(allRows.length)} results + ${_i('rankings.of')} ${fmtNum(allRows.length)} ${_i('rankings.results')} - Sorted by + ${_i('rankings.sorted')} ${esc(colLabel(cols, sortKey))} ${sortDir === "asc" ? "↑" : "↓"} @@ -158,8 +159,8 @@ export function render({ el, query }) { Showing only ${esc(chipFocusLabel)}${chipFocusVariants > 1 ? ` (across ${chipFocusVariants} chip-count variants)` : ""} in Suite ${esc(meta.letter)}. - View chip overview - Show all results + ${_i('rankings.viewchip')} + ${_i('rankings.showall')}
` : ""} @@ -167,11 +168,11 @@ export function render({ el, query }) {
${fmtNum(basketGet().length)} - ${basketGet().length === 1 ? "run" : "runs"} in your compare basket + ${basketGet().length === 1 ? _i('rankings.basket.run') : _i('rankings.basket.runs')} ${_i('rankings.basket.msg')} - Open compare → - + ${_i('rankings.basket.compare')} +
@@ -319,10 +320,10 @@ function renderTable(suiteId, rows, cols, sortKey, sortDir) { data-sort-key="date" data-sort-dir-default="desc" aria-sort="${sortKey === "date" ? (sortDir === "asc" ? "ascending" : "descending") : "none"}" scope="col"> - Date + ${_i('rankings.table.date')} ${sortKey === "date" ? (sortDir === "asc" ? "↑" : "↓") : ""} - Tier + ${_i('rankings.table.tier')} @@ -339,6 +340,7 @@ function renderRow(suiteId, row, cols, sortKey, rank) { const inBasket = basketHas(runId); const ver = shortVersion(row.framework_version); const fw = row.framework || ""; + const ops = implOpsLabel(row); // a11y: tabindex on the row makes the run-detail trigger reachable // via keyboard. We deliberately keep the native `` role so // assistive tech still announces row context (column → cell mapping @@ -361,7 +363,7 @@ function renderRow(suiteId, row, cols, sortKey, rank) { ${rank} - ${esc(row._chip_label)}${fw ? ` · ${esc(fw)}${ver ? ` ${esc(ver)}` : ""}` : ""} + ${esc(row._chip_label)}${fw ? ` · ${esc(fw)}${ver ? ` ${esc(ver)}` : ""}${ops ? ` ${esc(ops)}` : ""}` : ""} ${row.memory_gb ? `${esc(fmtNum(row.memory_gb))} GB` : ""} @@ -396,12 +398,12 @@ function renderEmpty(meta, filtersActive) { return `
-

No submissions yet for Suite ${esc(meta.letter)} · ${esc(meta.title)}.

-

${esc(meta.tagline)}

+

${_i('rankings.noSubmissionsFor')} Suite ${esc(meta.letter)} · ${esc(_i('suite.' + suiteId + '.title', meta.title))}.

+

${esc(_i('suite.' + suiteId + '.tagline', meta.tagline))}

`; } - // Resolve the active suite from the URL so the "Clear filters" href + // Resolve the active suite from the URL so the "${_i('rankings.clearFilters')}" href // strips facets while keeping the user on this same suite. const { params: q } = parseHash(location.hash); const sid = SUITE_ORDER.includes(q.suite) ? q.suite : SUITE_ORDER[0]; @@ -409,7 +411,7 @@ function renderEmpty(meta, filtersActive) {
`; } diff --git a/leaderboard/site/assets/js/views/reproduce.js b/leaderboard/site/assets/js/views/reproduce.js index ca37a64..c8cbcb5 100644 --- a/leaderboard/site/assets/js/views/reproduce.js +++ b/leaderboard/site/assets/js/views/reproduce.js @@ -1,7 +1,5 @@ // reproduce.js — Reproduction quests (community → verified), filterable table. - - import { esc, fmtNum, fmtDate, shortVersion } from "../utils.js"; import { rowByRunId, SUITE_META } from "../data.js"; @@ -19,8 +17,7 @@ import { import { discussResultUrl } from "../cite.js"; import { communityHeader, communityFilterRow, wireTableFilters } from "../community-nav.js"; - - +const _i = (k, r) => (window._i ? window._i(k, r) : k); function suiteLabel(id) { @@ -30,8 +27,6 @@ function suiteLabel(id) { } - - function recipeLabel(row) { if (!row?.framework) return "—"; @@ -44,69 +39,55 @@ function recipeLabel(row) { } - - function searchKey(q) { return `${q.chip} ${q.vendor} ${q.suite} ${q.framework || ""} ${q.submitted_by} ${q.run_id}`.toLowerCase(); } - - function renderCriteria(ex) { - const questWord = ex.openQuestCount === 1 ? "quest" : "quests"; + const questWord = ex.openQuestCount === 1 ? _i("reproduce.quest") : _i("reproduce.quests"); - const pairWord = ex.openPairCount === 1 ? "pair" : "pairs"; + const pairWord = ex.openPairCount === 1 ? _i("reproduce.pair") : _i("reproduce.pairs"); - const recordWord = ex.crossVerifiedRecordCount === 1 ? "record" : "records"; + const recordWord = ex.crossVerifiedRecordCount === 1 ? _i("reproduce.record") : _i("reproduce.records"); return ` `; @@ -566,9 +519,9 @@ function renderRoofline() { function renderInversion(card) { return `
- ${esc(card.eyebrow)} -

${esc(card.title)}

-

${esc(card.body)}

+ ${esc(_i('suites.inversion.' + card.suite + '.eyebrow'))} +

${esc(_i('suites.inversion.' + card.suite + '.title'))}

+

${esc(_i('suites.inversion.' + card.suite + '.body'))}

`; } @@ -583,20 +536,20 @@ function renderScenarioCard(scn) {

${esc(scn.name)}

- ${esc(scn.role)} + ${esc(_i('suites.scenario.' + scn.name + '.role', scn.role))}
-

${esc(scn.description)}

+

${esc(_i('suites.scenario.' + scn.name + '.desc'))}

${scn.spec.map((row) => `
-
${esc(row.k)}
-
${esc(row.v)}
+
${esc(_i('suites.spec.' + row.k, row.k))}
+
${esc(row.k === 'Direction' ? _i('suites.spec.Direction.' + row.v) : row.v)}
`).join("")}
- Used by + ${_i('suites.usedBy')} ${defaults.map((l) => `${esc(l)}`).join("")} ${extras.length && defaults.length ? `·` : ""} @@ -628,8 +581,8 @@ function renderSuiteSpec(suiteId) { ${esc(meta.letter)}
Suite ${esc(meta.letter)} -

${esc(meta.title)}

-

${esc(meta.tagline)}

+

${esc(_i('suite.' + suiteId + '.title', meta.title))}

+

${esc(_i('suite.' + suiteId + '.tagline', meta.tagline))}

${esc(meta.primary.label)} @@ -639,31 +592,31 @@ function renderSuiteSpec(suiteId) {
- ${meta.description ? `

${esc(meta.description)}

` : `
`} + ${meta.description ? `

${esc(_i('suite.' + suiteId + '.desc'))}

` : `
`} ${finding ? ` ` : ""}
    -
  • Model${esc(shortModel(wl.model) || "-")}
  • -
  • Chips${esc(wl.chips || "-")}
  • -
  • Precision${esc(wl.precision || "-")}
  • -
  • Dataset${esc(wl.dataset || "-")}
  • -
  • Tokens (in / out)${esc(wl.inputTokens || "-")} / ${esc(wl.outputTokens || "-")}
  • -
  • Coverage${fmtNum(facts.submissions)} results · ${fmtNum(facts.chips)} chips
  • +
  • ${_i('suites.wlModel')}${esc(shortModel(wl.model) || "-")}
  • +
  • ${_i('suites.wlChips')}${esc(wl.chips || "-")}
  • +
  • ${_i('suites.wlPrecision')}${esc(wl.precision || "-")}
  • +
  • ${_i('suites.wlDataset')}${esc(wl.dataset || "-")}
  • +
  • ${_i('suites.wlTokens')}${esc(wl.inputTokens || "-")} / ${esc(wl.outputTokens || "-")}
  • +
  • ${_i('suites.coverage')}${fmtNum(facts.submissions)} results · ${fmtNum(facts.chips)} chips
- Protocols + ${_i('suites.protocols')}
    ${scenarios.map((s) => ` -
  • - ${esc(s.name)}${s.isExtra ? `extra` : ""} +
  • + ${esc(s.name)}${s.isExtra ? `${_i('suites.extra')}` : ""}
  • `).join("")}
@@ -671,7 +624,7 @@ function renderSuiteSpec(suiteId) { ${leaderScenarios.length ? `
- Current leaders + ${_i('suites.currentLeaders')}
    ${leaderScenarios.map((s) => renderLeaderRow(suiteId, s)).join("")}
@@ -681,12 +634,12 @@ function renderSuiteSpec(suiteId) {
@@ -701,7 +654,7 @@ function renderLeaderRow(suiteId, scn) { return `
  • ${esc(scn.name)} - No qualifying submissions yet + ${_i('suites.noSubmissions')}
  • `; } @@ -741,7 +694,7 @@ function renderDataset(d) { ${esc(d.prompts)} ${esc(d.inputP50)} ${esc(d.outputP50)} - ${esc(d.notes)} + ${esc(_i('suites.dataset.' + d.name + '.notes'))}
    `; } diff --git a/leaderboard/site/assets/js/views/wanted.js b/leaderboard/site/assets/js/views/wanted.js index b795302..77be6fd 100644 --- a/leaderboard/site/assets/js/views/wanted.js +++ b/leaderboard/site/assets/js/views/wanted.js @@ -1,18 +1,18 @@ // wanted.js — Hardware gaps from accelerator catalog (filterable table). - import { esc, fmtNum } from "../utils.js"; import { chipCloudData } from "../data.js"; import { wantedFromCatalog, tierLabel, VENDORS } from "../hardware-catalog.js"; import { communityHeader, communityFilterRow, wireTableFilters } from "../community-nav.js"; +const _i = (k, r) => (window._i ? window._i(k, r) : k); -const TIERS = [ - { id: "", label: "All tiers" }, - { id: "datacenter", label: "Datacenter" }, - { id: "cloud", label: "Cloud" }, - { id: "workstation", label: "Workstation" }, - { id: "consumer", label: "Consumer / edge" }, - { id: "edge", label: "Edge" }, -]; +function tierLabels() { return [ + { id: "", label: _i('wanted.allTiers') }, + { id: "datacenter", label: _i('wanted.datacenter') }, + { id: "cloud", label: _i('wanted.cloud') }, + { id: "workstation", label: _i('wanted.workstation') }, + { id: "consumer", label: _i('wanted.consumer') }, + { id: "edge", label: _i('wanted.edge') }, +]; } function searchKey(w) { return `${w.name} ${w.vendorLabel} ${w.vendor} ${w.tier || ""} ${w.memoryGb || ""}`.toLowerCase(); @@ -25,13 +25,14 @@ export function render({ query, el }) { const preVendor = query?.vendor || ""; const vendorOptions = [ - ``, + ``, ...VENDORS.filter((v) => v.id !== "Other").map((v) => { const sel = v.id === preVendor ? " selected" : ""; return ``; }), ].join(""); + const TIERS = tierLabels(); const tierOptions = TIERS.map((t) => `` ).join(""); @@ -39,26 +40,26 @@ export function render({ query, el }) { el.innerHTML = ` ${communityHeader( "wanted", - "Wanted hardware", - `Accelerators present in our catalog but not yet covered on the leaderboard (${fmtNum(open.length)} open, ${fmtNum(covered)} covered). ` + - `The first published result for a platform receives a First result attribution on the ` + - `contributor index.` + _i('wanted.title'), + `${_i('wanted.desc1')} (${fmtNum(open.length)} ${_i('wanted.open')}, ${fmtNum(covered)} ${_i('wanted.covered')}). ` + + `${_i('wanted.desc2')} ` + + `${_i('wanted.contribIndex')}.` )}
    ${communityFilterRow([ { - label: "Vendor", + label: _i('wanted.filterVendor'), html: ``, countId: "wanted-count", }, { - label: "Tier", + label: _i('wanted.filterTier'), html: ``, }, { - label: "Search", - html: ``, + label: _i('wanted.filterSearch'), + html: ``, }, ])} @@ -67,11 +68,11 @@ export function render({ query, el }) { - - - - - + + + + + @@ -84,13 +85,13 @@ export function render({ query, el }) { `).join("")}
    PlatformVendorTierMemoryRecommended suites${_i('wanted.thPlatform')}${_i('wanted.thVendor')}${_i('wanted.thTier')}${_i('wanted.thMemory')}${_i('wanted.thSuites')}
    ${w.memoryGb ? `${w.memoryGb} GB` : "—"} ${(w.suites || ["suite_A", "suite_F"]).map((s) => `${esc(s)}`).join(" ")} - Submit + ${_i('wanted.submit')}
    - ` : `

    All catalog platforms are covered. Propose additional hardware via GitHub Discussions.

    `} + ` : `

    ${_i('wanted.allCovered')}

    `}
    `; diff --git a/leaderboard/site/distribution.js b/leaderboard/site/distribution.js index 47255b8..70a4915 100644 --- a/leaderboard/site/distribution.js +++ b/leaderboard/site/distribution.js @@ -8541,79 +8541,6 @@ const DISTRIBUTION_SUBMISSIONS = [ "peak_memory_gb": null } }, - { - "id": "83e3ec26", - "chip": "NVIDIA H20-3e", - "chip_vendor": "NVIDIA", - "chip_count": 1, - "memory_gb": 140.4, - "suite": "suite_A", - "model": "Meta-Llama-3-8B-Instruct", - "model_full": "meta-llama/Meta-Llama-3-8B-Instruct", - "model_params_b": 8.0, - "precision": "BF16", - "effective_dtype": "bfloat16", - "framework": "SGLang", - "framework_version": "0.5.6", - "tier": "community", - "submitted_by": "Gong-K", - "date": "2026-06-25", - "reproduce_script": "runners/nvidia_sglang_c43a8309/runner.py", - "runner_id": "nvidia_sglang_c43a8309", - "scenarios": { - "offline": { - "throughput": 4342.21, - "metric_label": "tokens/sec", - "concurrency": 128, - "peak_memory_gb": null, - "is_valid": true - }, - "online": { - "throughput": 100, - "metric_label": "max valid QPS", - "concurrency": null, - "peak_memory_gb": null, - "is_valid": true - }, - "interactive": { - "throughput": 4342.21, - "metric_label": "tokens/sec", - "concurrency": 128, - "peak_memory_gb": null, - "is_valid": true - }, - "sustained": { - "throughput": 1272.2, - "metric_label": "tok/s (sustained mean)", - "concurrency": 8, - "peak_memory_gb": null, - "is_valid": true - }, - "speculative": { - "throughput": 613.8, - "metric_label": "tok/s (speculative)", - "concurrency": 128, - "peak_memory_gb": null, - "is_valid": true - }, - "burst": { - "throughput": 0.835, - "metric_label": "1 − degradation_ratio", - "concurrency": null, - "peak_memory_gb": null, - "is_valid": true - } - }, - "primary_scenario": "offline", - "primary_throughput": 4342.21, - "primary_metric_label": "tokens/sec", - "config": { - "concurrency": 128, - "batch_size": null, - "tensor_parallel": 1, - "peak_memory_gb": null - } - }, { "id": "4e0e6eba", "chip": "Tesla V100-PCIE-32GB", @@ -9546,53 +9473,6 @@ const DISTRIBUTION_GROUPS = [ "best_framework": "vLLM", "best_submitted_by": "JuhaoLiang1997" }, - { - "chip": "NVIDIA H20-3e", - "chip_vendor": "NVIDIA", - "suite": "suite_A", - "model": "Meta-Llama-3-8B-Instruct", - "submission_count": 2, - "best_throughput": 4342.21, - "median_throughput": 4342.21, - "min_throughput": 2297.65, - "max_throughput": 4342.21, - "stddev_throughput": 1445.72, - "scenario_summary": { - "offline": { - "count": 2, - "best_throughput": 4342.21, - "best_framework": "SGLang" - }, - "online": { - "count": 2, - "best_throughput": 100, - "best_framework": "SGLang" - }, - "interactive": { - "count": 2, - "best_throughput": 4342.21, - "best_framework": "SGLang" - }, - "sustained": { - "count": 2, - "best_throughput": 1272.2, - "best_framework": "SGLang" - }, - "speculative": { - "count": 2, - "best_throughput": 783.62, - "best_framework": "vLLM" - }, - "burst": { - "count": 2, - "best_throughput": 0.835, - "best_framework": "SGLang" - } - }, - "best_submission_id": "83e3ec26", - "best_framework": "SGLang", - "best_submitted_by": "Gong-K" - }, { "chip": "NVIDIA A100-SXM4-80GB", "chip_vendor": "NVIDIA", @@ -10123,6 +10003,53 @@ const DISTRIBUTION_GROUPS = [ "best_framework": "vLLM", "best_submitted_by": "JuhaoLiang1997" }, + { + "chip": "NVIDIA H20-3e", + "chip_vendor": "NVIDIA", + "suite": "suite_A", + "model": "Meta-Llama-3-8B-Instruct", + "submission_count": 1, + "best_throughput": 2297.65, + "median_throughput": 2297.65, + "min_throughput": 2297.65, + "max_throughput": 2297.65, + "stddev_throughput": null, + "scenario_summary": { + "offline": { + "count": 1, + "best_throughput": 2297.65, + "best_framework": "vLLM" + }, + "online": { + "count": 1, + "best_throughput": 5, + "best_framework": "vLLM" + }, + "interactive": { + "count": 1, + "best_throughput": 2297.65, + "best_framework": "vLLM" + }, + "sustained": { + "count": 1, + "best_throughput": 486.6, + "best_framework": "vLLM" + }, + "speculative": { + "count": 1, + "best_throughput": 783.62, + "best_framework": "vLLM" + }, + "burst": { + "count": 1, + "best_throughput": null, + "best_framework": "" + } + }, + "best_submission_id": "3f6269bb", + "best_framework": "vLLM", + "best_submitted_by": "JuhaoLiang1997" + }, { "chip": "NVIDIA H20-3e", "chip_vendor": "NVIDIA", diff --git a/leaderboard/site/index.html b/leaderboard/site/index.html index 3691078..8aa9952 100644 --- a/leaderboard/site/index.html +++ b/leaderboard/site/index.html @@ -11,6 +11,22 @@ + @@ -53,6 +69,7 @@