diff --git a/.github/ISSUE_TEMPLATE/community_submission.md b/.github/ISSUE_TEMPLATE/community_submission.md deleted file mode 100644 index 3860bc80..00000000 --- a/.github/ISSUE_TEMPLATE/community_submission.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: Community submission -about: Submit a benchmark result to the AccelMark leaderboard -title: "[Submission] — — " -labels: community-submission ---- - - - -```json -(paste the full contents of results/community//result.json here) -``` \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 1501a5c0..87fd51de 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,8 @@ blank_issues_enabled: false contact_links: - name: Live leaderboard - url: https://juhaoliang1997.github.io/AccelMark + url: https://freedomintelligence.github.io/AccelMark about: View all submitted benchmark results - name: Contributing guide - url: https://github.com/JuhaoLiang1997/AccelMark/blob/main/CONTRIBUTING.md + url: https://github.com/FreedomIntelligence/AccelMark/blob/main/CONTRIBUTING.md about: How to run and submit a benchmark diff --git a/.github/ISSUE_TEMPLATE/new_suite.md b/.github/ISSUE_TEMPLATE/new_suite.md index 4c358a18..d3104b71 100644 --- a/.github/ISSUE_TEMPLATE/new_suite.md +++ b/.github/ISSUE_TEMPLATE/new_suite.md @@ -14,7 +14,7 @@ assignees: '' merging. Full walk-through: DEVELOPMENT.md "Adding a new suite" - https://github.com/JuhaoLiang1997/AccelMark/blob/main/DEVELOPMENT.md + https://github.com/FreedomIntelligence/AccelMark/blob/main/DEVELOPMENT.md --> ## Why this suite? diff --git a/.github/workflows/process_submissions.yml b/.github/workflows/process_submissions.yml deleted file mode 100644 index 2a0077a1..00000000 --- a/.github/workflows/process_submissions.yml +++ /dev/null @@ -1,265 +0,0 @@ -name: Process Community Submission - -# Triggered when an issue is labeled "community-submission" -# (The OpenClaw Skill creates issues with this label) -on: - issues: - types: [labeled] - -jobs: - process: - name: Extract and validate submission from issue - # Only run for the specific label - if: github.event.label.name == 'community-submission' - runs-on: ubuntu-latest - permissions: - contents: write - issues: write - pull-requests: write - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies - run: pip install jsonschema numpy --quiet - - - name: Extract result JSON from issue body - id: extract - uses: actions/github-script@v7 - with: - script: | - const body = context.payload.issue.body; - const issueNumber = context.payload.issue.number; - - // Extract JSON from code block in issue body - const match = body.match(/```json\n([\s\S]*?)\n```/); - if (!match) { - core.setFailed('No JSON code block found in issue body'); - return; - } - - let result; - try { - result = JSON.parse(match[1]); - } catch (e) { - core.setFailed(`Invalid JSON in issue: ${e.message}`); - return; - } - - // Use run_name from result.json - computed deterministically by benchmark_runner. - // Falls back to constructing from parts for older results that predate this field. - let dirName = result.meta?.run_name; - if (!dirName) { - const chip = (result.chip?.name || 'unknown') - .toLowerCase() - .replace(/[^a-z0-9]+/g, '_') - .replace(/^_|_$/g, ''); - const count = result.chip?.count || 1; - const suite = result.suite_id || 'unknown'; - const runner = result.implementation_id || 'unknown'; - const runId = result.meta?.run_id || 'unknown'; - dirName = `${chip}x${count}_${suite}_${runner}_${runId}`; - } - - core.setOutput('dir_name', dirName); - core.setOutput('issue_number', issueNumber); - core.setOutput('result_json', JSON.stringify(result)); - core.setOutput('run_id', result.meta?.run_id || ''); - - console.log(`Submission directory: ${dirName}`); - - - name: Check for duplicate submission - id: duplicate_check - run: | - RUN_ID="${{ steps.extract.outputs.run_id }}" - - if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "unknown" ]; then - echo "No run_id in submission - skipping duplicate check" - echo "is_duplicate=false" >> $GITHUB_OUTPUT - exit 0 - fi - - # Search all existing community result.json files for same run_id - # NOTE: runs BEFORE Write result files so $DIR does not exist yet - EXISTING="" - for f in $(find results/community -name "result.json" 2>/dev/null); do - MATCH=$(python3 -c "import json,sys; r=json.load(open(sys.argv[1])); print('yes' if r.get('meta',{}).get('run_id')=='$RUN_ID' else '')" "$f" 2>/dev/null || echo "") - if [ "$MATCH" = "yes" ]; then - EXISTING="$f" - break - fi - done - - if [ -n "$EXISTING" ]; then - echo "Found existing result with run_id=$RUN_ID at: $EXISTING" - echo "existing_path=$EXISTING" >> $GITHUB_OUTPUT - echo "is_duplicate=true" >> $GITHUB_OUTPUT - else - echo "No duplicate found for run_id=$RUN_ID" - echo "is_duplicate=false" >> $GITHUB_OUTPUT - fi - - - name: Handle duplicate submission - if: steps.duplicate_check.outputs.is_duplicate == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - EXISTING="${{ steps.duplicate_check.outputs.existing_path }}" - ISSUE="${{ steps.extract.outputs.issue_number }}" - EXISTING_DIR=$(dirname "$EXISTING") - - # New scenarios from step output (file not written yet) - NEW_SCENARIOS=$(python3 -c "import json,sys; r=json.loads(sys.argv[1]); s=sorted(r.get('task',{}).get('scenarios_run',[])); print(','.join(s) if s else 'none')" '${{ steps.extract.outputs.result_json }}' 2>/dev/null || echo "unknown") - - OLD_SCENARIOS=$(python3 -c "import json,sys; r=json.load(open(sys.argv[1])); s=sorted(r.get('task',{}).get('scenarios_run',[])); print(','.join(s) if s else 'none')" "$EXISTING" 2>/dev/null || echo "unknown") - - echo "Existing scenarios: $OLD_SCENARIOS" - echo "New scenarios: $NEW_SCENARIOS" - - if [ "$NEW_SCENARIOS" = "$OLD_SCENARIOS" ]; then - gh issue comment "$ISSUE" \ - --body "Duplicate submission detected. A result with the same hardware, software, suite, and submitter already exists at \`${EXISTING_DIR}\` with scenarios: \`${OLD_SCENARIOS}\`. If this is intentional (e.g. after a framework update), update \`meta.framework_version\` in your result.json and resubmit." - echo "Rejected: duplicate with same scenarios" - exit 1 - else - gh issue comment "$ISSUE" \ - --body "Note: An existing result with the same run_id was found at \`${EXISTING_DIR}\` (scenarios: \`${OLD_SCENARIOS}\`). This new submission has scenarios: \`${NEW_SCENARIOS}\`. Continuing - this looks like an update." - fi - - - name: Write result files - run: | - DIR="results/community/${{ steps.extract.outputs.dir_name }}" - mkdir -p "$DIR" - - # Write result.json, set meta.log_file to null (local artifact, not submitted) - echo '${{ steps.extract.outputs.result_json }}' | python -m json.tool > "$DIR/result.json" - python3 -c "import json; f=open('$DIR/result.json'); r=json.load(f); f.close(); r.get('meta',{}).update({'log_file':None}); open('$DIR/result.json','w').write(json.dumps(r,indent=2))" - echo "meta.log_file set to null" - - # Create env_info.json - extract from environment block if present, else reconstruct - python3 -c " - import json, sys - - with open('$DIR/result.json') as f: - result = json.load(f) - - chip = result.get('chip', {}) - software = result.get('software', {}) - - env = result.get('environment') - if not env: - # fallback for old results without environment block - env = { - 'collected_at': result.get('meta', {}).get('date', '2026-01-01') + 'T00:00:00Z', - 'accelerators': [{ - 'index': 0, - 'name': chip.get('name', 'Unknown'), - 'memory_gb': chip.get('memory_gb', 0), - 'driver_version': software.get('driver_version', 'unknown'), - 'firmware_version': None, - }], - 'accelerator_topology': None, - 'cpu': { - 'model': 'unknown', - 'physical_cores': 1, - 'logical_cores': 1, - 'numa_nodes': 1, - }, - 'system_memory_gb': 0, - 'pcie_generation': 'unknown', - 'cpu_accelerator_bandwidth_gbs': None, - 'network_interfaces': None, - 'kernel_version': 'unknown', - 'runtime_version': software.get('runtime_version', 'unknown'), - } - print('env_info.json reconstructed from chip data') - else: - print('env_info.json extracted from result.json') - - with open('$DIR/env_info.json', 'w') as f: - json.dump(env, f, indent=2) - - # Create accuracy/accuracy.json from the accuracy block in result.json - import os - acc = result.get('accuracy') - if acc: - os.makedirs('$DIR/accuracy', exist_ok=True) - with open('$DIR/accuracy/accuracy.json', 'w') as f: - json.dump(acc, f, indent=2) - print('accuracy/accuracy.json created') - else: - print('No accuracy block in result.json - skipping accuracy.json') - - " - - - name: Run validation - id: validate - run: | - DIR="results/community/${{ steps.extract.outputs.dir_name }}" - if python runners/validate_submission.py --dir "$DIR"; then - echo "valid=true" >> $GITHUB_OUTPUT - else - echo "valid=false" >> $GITHUB_OUTPUT - fi - - - name: Commit files and create PR - if: steps.validate.outputs.valid == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - DIR="results/community/${{ steps.extract.outputs.dir_name }}" - BRANCH="submission/${{ steps.extract.outputs.dir_name }}" - ISSUE="${{ steps.extract.outputs.issue_number }}" - - # Configure git - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Create and switch to submission branch - git checkout -b "$BRANCH" - - # Stage the result files written in the previous step - # Note: run.log is gitignored intentionally - not committed - git add "$DIR/result.json" "$DIR/env_info.json" - # accuracy/accuracy.json may exist if accuracy data was in result.json - [ -f "$DIR/accuracy/accuracy.json" ] && git add "$DIR/accuracy/accuracy.json" || true - - # Commit - git commit -m "community: add submission from issue #${ISSUE}" - - # Delete remote branch if it exists from a previous failed run, then push fresh - git push origin --delete "$BRANCH" 2>/dev/null || true - git push origin "$BRANCH" - - # Generate summary table and build PR body file - python3 runners/gen_pr_summary.py "$DIR/result.json" > /tmp/pr_summary.md - - printf "Auto-generated from Issue #%s\n\n## Submission summary\n\n" "$ISSUE" > /tmp/pr_body.md - cat /tmp/pr_summary.md >> /tmp/pr_body.md - printf "\nCloses #%s" "$ISSUE" >> /tmp/pr_body.md - - # Create PR (or get existing one if already created) - PR_URL=$(gh pr create \ - --title "[community] ${{ steps.extract.outputs.dir_name }}" \ - --body-file /tmp/pr_body.md \ - --base main \ - --head "$BRANCH" 2>/dev/null) || \ - PR_URL=$(gh pr view "$BRANCH" --json url -q .url) - - # Comment on the issue with PR link - gh issue comment "$ISSUE" --body "Validation passed. PR created: ${PR_URL} - your result will appear on the leaderboard after the PR is reviewed and merged." - - - - name: Comment failure on issue - if: steps.validate.outputs.valid == 'false' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh issue comment "${{ steps.extract.outputs.issue_number }}" \ - --body "Validation failed. See the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. Fix the errors and ask a maintainer to re-apply the community-submission label to retry." \ No newline at end of file diff --git a/.github/workflows/validate_pr.yml b/.github/workflows/validate_pr.yml index 541af133..6ffe949f 100644 --- a/.github/workflows/validate_pr.yml +++ b/.github/workflows/validate_pr.yml @@ -14,6 +14,12 @@ on: - 'README.md' - 'leaderboard/site/**' +# Lock the default GITHUB_TOKEN down to read-only on repo contents for every +# job (GitHub's hardening recommendation for OSS repos). Jobs that need more +# (e.g. posting a PR comment) opt in explicitly via their own `permissions:`. +permissions: + contents: read + jobs: validate-runners: name: Validate runner folders @@ -116,6 +122,13 @@ jobs: validate: name: Validate result submissions runs-on: ubuntu-latest + # Needs pull-requests:write to post the validation summary comment + # via actions/github-script. After the FreedomIntelligence transfer + # the default GITHUB_TOKEN is read-only on issues/pull-requests, so + # this must be granted explicitly. + permissions: + contents: read + pull-requests: write steps: - name: Checkout PR diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 354654ba..a476a44e 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -75,7 +75,7 @@ appointed representative at an online or offline event. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project maintainers by opening a confidential security -advisory at +advisory at or, when GitHub access is not available, by emailing the maintainer listed in the repository profile. All complaints will be reviewed and investigated promptly and fairly. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a3044cdc..b582f8cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,8 +36,6 @@ gh pr create # or open the PR via the GitHub web UI That's it. CI validates the result automatically; merging the PR publishes it to the leaderboard. -> _Prefer not to use git?_ Open a [Community Submission issue](https://github.com/JuhaoLiang1997/AccelMark/issues/new?template=community_submission.md), paste your `result.json`, and the CI bot will draft the PR on your behalf. - --- ## One-time setup @@ -335,23 +333,13 @@ python -m http.server -d leaderboard/site 8000 # serve the static site Both `leaderboard.js` and `leaderboard/site/api/` are gitignored — the GitHub Actions workflow regenerates them on every merge to `main`. -### Alternative: open a submission issue (no git required) - -If you'd rather not use git, paste your `result.json` into a -[Community Submission issue](https://github.com/JuhaoLiang1997/AccelMark/issues/new?template=community_submission.md). -A bot will validate the JSON, draft a PR with the files in the right place, -and link it back to your issue. You don't need to touch git or fork the repo. - -> **Why paste instead of attach?** The bot reads `result.json` directly from -> the issue body. File attachments are not accessible to GitHub Actions. - --- ## Leaderboard tiers | Tier | How to get it | Leaderboard placement | |------|--------------|----------------------| -| **community** | Submit a PR (or issue → bot-drafted PR) and pass CI validation | Community tab | +| **community** | Submit a PR and pass CI validation | Community tab | | **verified** | Independently reproduced on the same hardware/runner within 5% | Main leaderboard | To promote a community result to **verified**, anyone with the same hardware diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 98cddbca..990f73bc 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1134,6 +1134,6 @@ python runners/validate_submission.py --dir /tmp/accelmark_test/ ## Questions and Support - **Bug in LoadGen or schema:** Open a GitHub Issue -- **New suite proposal:** Open a GitHub Issue with the [**Propose a new suite**](https://github.com/JuhaoLiang1997/AccelMark/issues/new?template=new_suite.md) template +- **New suite proposal:** Open a GitHub Issue with the [**Propose a new suite**](https://github.com/FreedomIntelligence/AccelMark/issues/new?template=new_suite.md) template - **New platform support:** Open a PR with a working platform script and at least one verified result - **Leaderboard question:** Check `leaderboard/generate.py` — it's well-commented \ No newline at end of file diff --git a/NOTICE b/NOTICE index d904b638..a0181123 100644 --- a/NOTICE +++ b/NOTICE @@ -2,7 +2,7 @@ AccelMark Copyright 2024-2026 Juhao Liang and The AccelMark Contributors This product includes software developed as part of the AccelMark project -(https://github.com/JuhaoLiang1997/AccelMark). +(https://github.com/FreedomIntelligence/AccelMark). Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 5fb2f691..c7a18a3e 100644 --- a/README.md +++ b/README.md @@ -10,16 +10,16 @@

- Live Leaderboard + Live Leaderboard License: Apache 2.0 Contributions welcome

- → Live Leaderboard · + → Live Leaderboard · Contributing · Suites · - Discussions · + Discussions · Development

@@ -49,7 +49,7 @@ ```bash # 1. Clone and install -git clone https://github.com/JuhaoLiang1997/AccelMark.git +git clone https://github.com/FreedomIntelligence/AccelMark.git cd AccelMark pip install -e . # installs framework dependencies (Python >=3.10 required) pip install -r runners/nvidia_vllm_47f5d58e/requirements.txt # installs runner dependencies @@ -70,7 +70,7 @@ python run.py --runner nvidia_vllm_47f5d58e --suite suite_A # your result.json and env_info.json; no manual file moves are needed. ``` -See [CONTRIBUTING.md](CONTRIBUTING.md) for the full guide. If you'd rather skip the PR workflow, [open a submission issue](https://github.com/JuhaoLiang1997/AccelMark/issues/new?template=community_submission.md) instead and a bot will draft the PR for you. +See [CONTRIBUTING.md](CONTRIBUTING.md) for the full guide. --- @@ -100,7 +100,7 @@ See [suites/README.md](suites/README.md) for full specs, time budgets, SLA defin width="980">

-A snapshot of accelerators that have at least one submission on the leaderboard. Tile size is proportional to submission count; colour denotes vendor. See the [**live leaderboard**](https://juhaoliang1997.github.io/AccelMark) for current rankings, per-suite breakdowns, and the underlying `result.json` files. +A snapshot of accelerators that have at least one submission on the leaderboard. Tile size is proportional to submission count; colour denotes vendor. See the [**live leaderboard**](https://freedomintelligence.github.io/AccelMark) for current rankings, per-suite breakdowns, and the underlying `result.json` files. --- @@ -136,7 +136,7 @@ Adding a new runner? See [CONTRIBUTING.md#adding-a-new-runner](CONTRIBUTING.md#a | Tier | How | Where | |------|-----|-------| -| **community** | Submitted by anyone via PR (or issue → bot-drafted PR) and passes CI validation | Community tab | +| **community** | Submitted by anyone via PR and passes CI validation | Community tab | | **verified** | Independently reproduced on the same hardware/runner and matches the original within 5% | Main leaderboard | Community results are fully visible and comparable — they just haven't been independently reproduced yet. Anyone with the listed hardware can promote a community result to verified by submitting a reproduction PR. @@ -150,8 +150,8 @@ The most valuable contribution is running the benchmark on hardware not yet in t - **Submit a result** → [Submitting a result](CONTRIBUTING.md#submitting-a-result) - **Add a new runner** → [Adding a new runner](CONTRIBUTING.md#adding-a-new-runner) - **Add a new accelerator family** → [Platform plug-in guide](runners/README.md#adding-a-new-accelerator-family) -- **Report a bug** → [Open an issue](https://github.com/JuhaoLiang1997/AccelMark/issues/new?template=bug_report.md) -- **Ask a question / share results** → [Discussions](https://github.com/JuhaoLiang1997/AccelMark/discussions) +- **Report a bug** → [Open an issue](https://github.com/FreedomIntelligence/AccelMark/issues/new?template=bug_report.md) +- **Ask a question / share results** → [Discussions](https://github.com/FreedomIntelligence/AccelMark/discussions) - **Extend the leaderboard** → [Development guide](DEVELOPMENT.md) > _Optional:_ AccelMark also ships a small voice-driven launcher for the [OpenClaw](https://clawhub.ai) ecosystem — see [`openclaw_skill/`](openclaw_skill/README.md). It's not required to run, contribute, or submit results. @@ -167,7 +167,7 @@ If you use AccelMark results in research, please cite: title = {Beyond NVIDIA! A Multi-Regime Framework for Benchmarking Heterogeneous AI Accelerators}, author = {Liang, Juhao and Zhang, Zhiyuan and Li, Siyu and Lin, Zhihang and Yu, Minchen and Zeng, Li and Chen, Zizhong and Sun, Ruoyu and Wang, Benyou}, year = {2026}, - url = {https://github.com/JuhaoLiang1997/AccelMark} + url = {https://github.com/FreedomIntelligence/AccelMark} } ``` diff --git a/SECURITY.md b/SECURITY.md index 67be232b..11d96f5b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -40,7 +40,7 @@ providing an initial assessment. We aim to publish a fix and credit the reporter within **30 days** of acknowledgement; if a fix is going to take longer we will say so in the response. -[advisory]: https://github.com/JuhaoLiang1997/AccelMark/security/advisories/new +[advisory]: https://github.com/FreedomIntelligence/AccelMark/security/advisories/new When reporting, please include: diff --git a/leaderboard/generate.py b/leaderboard/generate.py index 4a9ac5ab..bf35de21 100644 --- a/leaderboard/generate.py +++ b/leaderboard/generate.py @@ -273,8 +273,8 @@ def extract_impl(result: dict) -> dict | None: "created": meta.get("created"), "supersedes_chain": meta.get("supersedes_chain"), "deprecated_by": meta.get("deprecated_by"), - "github_url": f"https://github.com/JuhaoLiang1997/AccelMark/tree/main/runners/{impl_id}", - "runner_url": f"https://github.com/JuhaoLiang1997/AccelMark/blob/main/runners/{impl_id}/runner.py", + "github_url": f"https://github.com/FreedomIntelligence/AccelMark/tree/main/runners/{impl_id}", + "runner_url": f"https://github.com/FreedomIntelligence/AccelMark/blob/main/runners/{impl_id}/runner.py", } diff --git a/leaderboard/site/assets/js/modal.js b/leaderboard/site/assets/js/modal.js index a52a3e1f..3461dbbf 100644 --- a/leaderboard/site/assets/js/modal.js +++ b/leaderboard/site/assets/js/modal.js @@ -306,7 +306,7 @@ function _fillModal(row) { const impl = row.impl || {}; let scriptUrl = null; if (d.meta_reproduce_script) { - scriptUrl = `https://github.com/JuhaoLiang1997/AccelMark/blob/main/${d.meta_reproduce_script}`; + scriptUrl = `https://github.com/FreedomIntelligence/AccelMark/blob/main/${d.meta_reproduce_script}`; // If the result references an old (superseded) runner path, point to // the current runner.py instead. if (impl.supersedes_chain && Array.isArray(impl.supersedes_chain)) { diff --git a/leaderboard/site/assets/js/views/home.js b/leaderboard/site/assets/js/views/home.js index 32744317..94602437 100644 --- a/leaderboard/site/assets/js/views/home.js +++ b/leaderboard/site/assets/js/views/home.js @@ -107,11 +107,8 @@ export function render({ el }) {

Contributor guide → - Open a submission
diff --git a/leaderboard/site/assets/js/views/suites.js b/leaderboard/site/assets/js/views/suites.js index 15e22ae4..f1177b65 100644 --- a/leaderboard/site/assets/js/views/suites.js +++ b/leaderboard/site/assets/js/views/suites.js @@ -7,7 +7,7 @@ import { } from "../data.js"; import { esc, fmtNum, buildHash, shortModel } from "../utils.js"; -const GH_BASE = "https://github.com/JuhaoLiang1997/AccelMark"; +const GH_BASE = "https://github.com/FreedomIntelligence/AccelMark"; // One concrete finding per suite, distilled from the paper. Kept short // enough to fit inside a single card but specific enough to be useful. diff --git a/leaderboard/site/index.html b/leaderboard/site/index.html index 10013843..3ccc2802 100644 --- a/leaderboard/site/index.html +++ b/leaderboard/site/index.html @@ -46,7 +46,7 @@
- + @@ -61,9 +61,9 @@ diff --git a/openclaw_skill/accelmark_skill.py b/openclaw_skill/accelmark_skill.py index 698bb937..7f55cc18 100644 --- a/openclaw_skill/accelmark_skill.py +++ b/openclaw_skill/accelmark_skill.py @@ -10,9 +10,13 @@ 3. Runs the benchmark (5 min max) 4. Queries AccelMark leaderboard for ranking 5. Returns human-readable report to user -6. Optionally submits result to community leaderboard +6. On "submit", returns copy-pasteable PR submission instructions + (AccelMark uses a PR-only submission flow; the skill no longer posts + to GitHub directly) """ +from __future__ import annotations + import json import os import subprocess @@ -22,7 +26,8 @@ ACCELMARK_REPO = os.environ.get("ACCELMARK_PATH", "~/accelmark") -LEADERBOARD_API = "https://juhaoliang1997.github.io/AccelMark/api" +LEADERBOARD_API = "https://freedomintelligence.github.io/AccelMark/api" +REPO_URL = "https://github.com/FreedomIntelligence/AccelMark" def ensure_accelmark_installed() -> Path: @@ -31,7 +36,7 @@ def ensure_accelmark_installed() -> Path: if not repo_path.exists(): subprocess.run( - ["git", "clone", "https://github.com/JuhaoLiang1997/AccelMark.git", str(repo_path)], + ["git", "clone", f"{REPO_URL}.git", str(repo_path)], check=True ) @@ -102,49 +107,39 @@ def query_submission_rank(submission_name: str) -> dict | None: def handle_submit(result: dict, openclaw_username: str) -> str: """ - Submit result to AccelMark leaderboard via GitHub Issue. - The issue is processed by the process_submissions.yml CI workflow - which validates and creates a PR automatically. - """ - import urllib.request + Format submission instructions for the user. + AccelMark uses a pull-request-only submission flow — bot-driven + automation via GitHub Issues was removed because it required org-level + permissions and added a hard-to-audit attack surface. This function + returns the finalised result.json (with ``submitted_by`` set) and the + copy-pasteable steps the user needs to follow to open a PR themselves. + """ result["meta"]["submitted_by"] = openclaw_username - - issue_body = f"""## AccelMark Community Submission - -**Chip**: {result['chip']['name']} -**Suite**: {result['suite_id']} -**Date**: {result['meta']['date']} -**Submitted via**: OpenClaw AccelMark Skill - -```json -{json.dumps(result, indent=2)} -``` -""" - payload = json.dumps({ - "title": f"[submission] {result['chip']['name']} {result['suite_id']}", - "body": issue_body, - "labels": ["community-submission"], - }).encode() - - req = urllib.request.Request( - "https://api.github.com/repos/JuhaoLiang1997/AccelMark/issues", - data=payload, - headers={"Content-Type": "application/json"}, + run_name = ( + result.get("meta", {}).get("run_name") + or f"{result.get('chip', {}).get('name', 'unknown')}_{result.get('suite_id', 'unknown')}" + ) + result_json = json.dumps(result, indent=2) + + return ( + "📦 Your benchmark result is ready to submit.\n" + "\n" + "AccelMark uses a pull-request-only submission flow. To add this\n" + "result to the public leaderboard:\n" + "\n" + f" 1. Fork and clone {REPO_URL}\n" + f" 2. mkdir -p results/community/{run_name}\n" + f" 3. Save the JSON below as results/community/{run_name}/result.json\n" + " 4. Commit, push, and open a pull request — CI auto-validates.\n" + "\n" + f"Full guide: {REPO_URL}/blob/main/CONTRIBUTING.md\n" + "\n" + "--- result.json ---\n" + "```json\n" + f"{result_json}\n" + "```" ) - - try: - urllib.request.urlopen(req, timeout=10) - return ( - "✅ Submitted! Your result will appear on the leaderboard " - "after automated validation (usually within a few minutes).\n" - f"View leaderboard: {LEADERBOARD_API.replace('/api', '/')}" - ) - except Exception as e: - return ( - f"❌ Submission failed: {str(e)}\n" - "You can submit manually at: https://github.com/JuhaoLiang1997/AccelMark" - ) def format_details(result: dict) -> str: diff --git a/openclaw_skill/tests/test_accelmark_skill.py b/openclaw_skill/tests/test_accelmark_skill.py index d0a2a7a0..b09f0010 100644 --- a/openclaw_skill/tests/test_accelmark_skill.py +++ b/openclaw_skill/tests/test_accelmark_skill.py @@ -4,7 +4,8 @@ Covers the skill entry point functions — no real GPU, no real network. All external calls (urllib, subprocess) are mocked. - main(): follow-up flows for "submit" and "details" -- handle_submit(): GitHub Issue posting, success and failure paths +- handle_submit(): renders PR submission instructions (the skill no + longer auto-creates a GitHub Issue — AccelMark is PR-only) - query_ranking(): graceful None on any network error - format_details(): table rendering, ← best marker - _summarize_chips(): GPU env and CPU-only env @@ -99,36 +100,38 @@ def test_query_ranking_returns_none_for_unknown_chip(): # ── handle_submit ───────────────────────────────────────────────────────────── -def test_handle_submit_success(): - mock_resp = MagicMock() - mock_resp.__enter__ = lambda s: s - mock_resp.__exit__ = MagicMock(return_value=False) - - with patch("urllib.request.urlopen", return_value=mock_resp): - msg = handle_submit(_make_result(), openclaw_username="testuser") +def test_handle_submit_returns_pr_instructions(): + msg = handle_submit(_make_result(), openclaw_username="testuser") - assert "✅" in msg - assert "leaderboard" in msg.lower() + # Instructions, not auto-submission confirmation + assert "pull request" in msg.lower() or "pr" in msg.lower() + assert "results/community/" in msg + assert "CONTRIBUTING.md" in msg def test_handle_submit_sets_submitted_by(): result = _make_result() - mock_resp = MagicMock() - mock_resp.__enter__ = lambda s: s - mock_resp.__exit__ = MagicMock(return_value=False) + handle_submit(result, openclaw_username="alice") + assert result["meta"]["submitted_by"] == "alice" - with patch("urllib.request.urlopen", return_value=mock_resp): - handle_submit(result, openclaw_username="alice") - assert result["meta"]["submitted_by"] == "alice" +def test_handle_submit_embeds_result_json(): + result = _make_result(chip_name="NVIDIA H100") + msg = handle_submit(result, openclaw_username="bob") + # The full result.json is embedded in a ```json fenced block so the + # user can copy it directly into their PR. + assert "```json" in msg + assert "NVIDIA H100" in msg + assert '"submitted_by": "bob"' in msg -def test_handle_submit_failure_returns_error_message(): - with patch("urllib.request.urlopen", side_effect=Exception("connection refused")): - msg = handle_submit(_make_result(), openclaw_username="bob") - assert "❌" in msg - assert "manually" in msg.lower() or "github" in msg.lower() +def test_handle_submit_makes_no_network_call(): + # Regression: this function used to POST to api.github.com. The new + # implementation must be purely local — no urllib usage at all. + with patch("urllib.request.urlopen") as mock_urlopen: + handle_submit(_make_result(), openclaw_username="carol") + mock_urlopen.assert_not_called() # ── format_details ──────────────────────────────────────────────────────────── @@ -163,14 +166,11 @@ def test_main_details_without_pending_result(): def test_main_submit_with_pending_result(): ctx = {"accelmark_pending_result": _make_result(), "user": {"username": "carol"}} - mock_resp = MagicMock() - mock_resp.__enter__ = lambda s: s - mock_resp.__exit__ = MagicMock(return_value=False) - - with patch("urllib.request.urlopen", return_value=mock_resp): - msg = main("submit", ctx) + msg = main("submit", ctx) - assert "✅" in msg + # New flow: hand the user PR instructions instead of auto-posting. + assert "pull request" in msg.lower() or "pr" in msg.lower() + assert "CONTRIBUTING.md" in msg def test_main_details_with_pending_result(): diff --git a/pyproject.toml b/pyproject.toml index bbdcf567..62a7585e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,12 +48,12 @@ dependencies = [ ] [project.urls] -Homepage = "https://github.com/JuhaoLiang1997/AccelMark" -Leaderboard = "https://juhaoliang1997.github.io/AccelMark" -Documentation = "https://github.com/JuhaoLiang1997/AccelMark/blob/main/README.md" -Repository = "https://github.com/JuhaoLiang1997/AccelMark" -Issues = "https://github.com/JuhaoLiang1997/AccelMark/issues" -Changelog = "https://github.com/JuhaoLiang1997/AccelMark/releases" +Homepage = "https://github.com/FreedomIntelligence/AccelMark" +Leaderboard = "https://freedomintelligence.github.io/AccelMark" +Documentation = "https://github.com/FreedomIntelligence/AccelMark/blob/main/README.md" +Repository = "https://github.com/FreedomIntelligence/AccelMark" +Issues = "https://github.com/FreedomIntelligence/AccelMark/issues" +Changelog = "https://github.com/FreedomIntelligence/AccelMark/releases" [build-system] requires = ["setuptools>=68"] diff --git a/runners/gen_pr_summary.py b/runners/gen_pr_summary.py deleted file mode 100644 index e7c056f0..00000000 --- a/runners/gen_pr_summary.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate a markdown summary table from a result.json file. -Used by .github/workflows/process_submissions.yml to build PR descriptions. -Usage: python runners/gen_pr_summary.py -""" -import json, sys - -with open(sys.argv[1]) as f: - r = json.load(f) - -chip = r.get('chip', {}) -model = r.get('model', {}) -sw = r.get('software', {}) -metrics = r.get('metrics', {}) -acc = r.get('accuracy') or {} -meta = r.get('meta', {}) -suite = r.get('suite_id', '') -dash = '—' - -# ── Primary throughput — varies by suite ───────────────────────────────────── -thr = None - -# Suite A/B/D: standard offline block -offline = metrics.get('offline', {}) -rows = (offline.get('results_by_concurrency') or - offline.get('results_by_batch_size') or []) -valid = [x for x in rows if not x.get('oom') and x.get('throughput_tokens_per_sec')] -if valid: - thr = round(max(x['throughput_tokens_per_sec'] for x in valid)) - -# Suite C: best throughput across precision formats -if thr is None: - quant = metrics.get('quantization', {}) - entries = quant.get('results_by_precision', []) - vals = [e.get('best_throughput_tokens_per_sec') for e in entries - if e.get('best_throughput_tokens_per_sec')] - if vals: - thr = round(max(vals)) - -# Suite E: base throughput from scaling block -if thr is None: - scaling = metrics.get('scaling', {}) - base = scaling.get('base_throughput_tokens_per_sec') - if base: - thr = round(base) - else: - for entry in scaling.get('results_by_chip_count', []): - if entry.get('chip_count') == 1: - t = entry.get('best_throughput_tokens_per_sec') - if t: - thr = round(t) - break - -# ── Online / interactive ────────────────────────────────────────────────────── -online = metrics.get('online', {}) -iv = metrics.get('interactive', {}) -qps = online.get('max_valid_qps') -ttft = iv.get('ttft_ms_p99') -# Treat 0 or None as no result -if not qps: - qps = None - -# ── Suite C: accuracy per format ───────────────────────────────────────────── -if suite == 'suite_C': - quant = metrics.get('quantization', {}) - entries = quant.get('results_by_precision', []) - bf16 = next((e for e in entries if e.get('precision') == 'BF16'), {}) - acc_val = f"{bf16.get('accuracy_score', dash)} (BF16)" if bf16 else dash -elif suite == 'suite_E': - acc_val = dash # Suite E does not run accuracy -elif acc: - acc_val = acc.get('valid', dash) -else: - acc_val = dash - -# ── Suite E: scaling efficiency ─────────────────────────────────────────────── -scaling_note = '' -if suite == 'suite_E': - scaling = metrics.get('scaling', {}) - parts = [] - for entry in scaling.get('results_by_chip_count', []): - count = entry.get('chip_count') - eff = entry.get('scaling_efficiency') - if count and count > 1 and eff is not None: - parts.append(f'{count}×: {round(eff * 100)}%') - scaling_note = ', '.join(parts) if parts else dash - -# ── Print table ─────────────────────────────────────────────────────────────── -print('| Field | Value |') -print('|---|---|') -print(f"| Chip | {chip.get('name', dash)} |") -print(f"| Chip count | {chip.get('count', 1)} |") -print(f"| Suite | {suite or dash} |") -print(f"| Runner | {r.get('implementation_id', dash)} |") -print(f"| Framework | {sw.get('framework', dash)} {sw.get('framework_version', '')} |") -print(f"| Model | {model.get('model_id', dash)} |") -print(f"| Precision | {model.get('precision', dash)} |") -print(f"| Offline throughput | {thr if thr else dash} tok/s |") -if suite == 'suite_E' and scaling_note: - print(f"| Scaling efficiency | {scaling_note} |") -else: - print(f"| Online max QPS | {round(qps, 1) if qps is not None else dash} |") - print(f"| Interactive TTFT p99 | {round(ttft, 1) if ttft is not None else dash} ms |") -print(f"| Accuracy valid | {acc_val} |") -print(f"| Submitted by | {meta.get('submitted_by', dash)} |") -date_str = meta.get('date', dash) -time_str = meta.get('time', '') -print(f"| Date | {date_str}{' ' + time_str if time_str else ''} |") -print(f"| Run ID | `{meta.get('run_id', dash)}` |") -print(f"| Run name | `{meta.get('run_name', dash)}` |") -if meta.get('flagged'): - print(f"| ⚠️ Flagged | {meta['flagged']} |") \ No newline at end of file diff --git a/runners/validate_runners.py b/runners/validate_runners.py index 14db0a9a..5cb350ac 100644 --- a/runners/validate_runners.py +++ b/runners/validate_runners.py @@ -49,8 +49,8 @@ "collect_env.py", "validate_submission.py", "validate_runners.py", + "validate_suites.py", "hash_runner.py", - "gen_pr_summary.py", "meta.schema.json", "protocol.py", "template", diff --git a/suites/README.md b/suites/README.md index aedb5652..24d7c95b 100644 --- a/suites/README.md +++ b/suites/README.md @@ -716,7 +716,7 @@ submissions. ## Adding a new suite -1. Open a GitHub Issue using the [**Propose a new suite**](https://github.com/JuhaoLiang1997/AccelMark/issues/new?template=new_suite.md) template +1. Open a GitHub Issue using the [**Propose a new suite**](https://github.com/FreedomIntelligence/AccelMark/issues/new?template=new_suite.md) template 2. Specify: model, chip count, scenarios, and rationale 3. Discuss the proposal in the issue thread — interested contributors weigh in 4. Create `suites/suite_X/suite.json` referencing a shared dataset diff --git a/tools/generate_platforms_matrix.py b/tools/generate_platforms_matrix.py index 5e24b126..fc104b3c 100644 --- a/tools/generate_platforms_matrix.py +++ b/tools/generate_platforms_matrix.py @@ -40,8 +40,8 @@ "collect_env.py", "validate_submission.py", "validate_runners.py", + "validate_suites.py", "hash_runner.py", - "gen_pr_summary.py", "meta.schema.json", "protocol.py", "template",