Make ART a native multi-node RL runtime with Monarch

.github/workflows/package-install.yml

@@ -2,15 +2,14 @@ name: Package Install
 
 on:
   pull_request:
-  push:
     branches: [main]
-  workflow_dispatch:
 
 permissions:
   contents: read
 
 jobs:
   install-smoke-test:
+    if: startsWith(github.head_ref, 'release/')
     runs-on: ubuntu-latest
 
     steps:
@@ -26,7 +25,7 @@ jobs:
           curl -LsSf https://astral.sh/uv/install.sh | sh
           echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
 
-      - name: Build wheel
+      - name: Build managed-runtime wheel
         run: python scripts/build_package.py --wheel
 
       - name: Smoke test base wheel import
@@ -42,7 +41,7 @@ jobs:
           cd "$project_dir"
           uv init --name art-base-install-smoke --python 3.12 --bare
           uv add "openpipe-art @ file://${wheel_path}"
-          uv run python -c "import importlib.util; assert importlib.util.find_spec('numpy') is None; assert importlib.util.find_spec('torch') is None; import art; from art.pipeline_trainer import PipelineTrainer; print(art.__name__, PipelineTrainer.__name__)"
+          uv run python -c "import importlib.util; assert importlib.util.find_spec('numpy') is not None; assert importlib.util.find_spec('torch') is None; import art; from art import ServerlessBackend; from art.pipeline_trainer import PipelineTrainer; print(art.__name__, ServerlessBackend.__name__, PipelineTrainer.__name__)"
           uv add "weave==0.52.41"
           uv run python -c "from weave.trace.settings import override_settings; import art"
 
@@ -60,3 +59,140 @@ jobs:
           uv init --name art-install-smoke --python 3.12 --bare
           uv add "openpipe-art[backend] @ file://${wheel_path}"
           uv sync
+
+      - name: Resolve published Megatron profiles
+        run: |
+          wheel_path="$(python - <<'PY'
+          from pathlib import Path
+
+          print(next(Path("dist").glob("openpipe_art-*.whl")).resolve())
+          PY
+          )"
+
+          for profile in megatron megatron-cu130; do
+            case "$profile" in
+              megatron) index=cu128 ;;
+              megatron-cu130) index=cu130 ;;
+            esac
+            project_dir="$(mktemp -d)"
+            cd "$project_dir"
+            uv init --name "art-${profile}-resolve" --python 3.12 --bare
+            uv add --no-sync \
+              --index-strategy unsafe-best-match \
+              --extra-index-url "https://download.pytorch.org/whl/${index}" \
+              "openpipe-art[${profile}] @ file://${wheel_path}"
+          done
+
+      - name: Resolve published Tinker profile
+        run: |
+          wheel_path="$(python - <<'PY'
+          from pathlib import Path
+
+          print(next(Path("dist").glob("openpipe_art-*.whl")).resolve())
+          PY
+          )"
+
+          project_dir="$(mktemp -d)"
+          cd "$project_dir"
+          uv init --name art-tinker-resolve --python 3.12 --bare
+          uv add --no-sync "openpipe-art[tinker] @ file://${wheel_path}"
+
+      - name: Smoke test distributed wheel surface on CPU
+        env:
+          ART_VLLM_RUNTIME_CACHE_DIR: ${{ runner.temp }}/art-vllm-runtime-cache
+        run: |
+          wheel_path="$(python - <<'PY'
+          from pathlib import Path
+
+          print(next(Path("dist").glob("openpipe_art-*.whl")).resolve())
+          PY
+          )"
+
+          project_dir="$(mktemp -d)"
+          cd "$project_dir"
+          uv init --name art-distributed-install-smoke --python 3.12 --bare
+          uv venv --python 3.12
+          uv pip install --python .venv/bin/python \
+            "openpipe-art @ file://${wheel_path}"
+          uv pip install --python .venv/bin/python \
+            --index-url https://download.pytorch.org/whl/cpu \
+            "torch==2.11.0"
+          uv pip install --python .venv/bin/python \
+            "msgspec>=0.21.0" \
+            "torchmonarch==0.6.0" \
+            "transformers==5.12.1"
+
+          .venv/bin/python - <<'PY'
+          import sys
+          from importlib.metadata import metadata
+          from pathlib import Path
+          import subprocess
+          import tempfile
+          import venv
+
+          import art
+          import art.distributed as distributed
+
+          assert Path(art.__file__).resolve().is_relative_to(Path(sys.prefix))
+          assert "art.distributed.art_runtime" not in sys.modules
+          from art.distributed import (  # noqa: E402
+              ArtRuntime,
+              ClusterSpec,
+              HostSpec,
+              InstalledAsyncCallable,
+              NcclTransportSpec,
+              PackingRequest,
+              compile_topology,
+          )
+
+          assert all(
+              value is not None
+              for value in (
+                  ArtRuntime,
+                  ClusterSpec,
+                  HostSpec,
+                  InstalledAsyncCallable,
+                  NcclTransportSpec,
+                  PackingRequest,
+                  compile_topology,
+              )
+          )
+          assert "monarch" not in sys.modules
+          assert "PackingRequest" in distributed.__all__
+          assert "NcclTransportSpec" in distributed.__all__
+          assert {"distributed", "megatron"} <= set(
+              metadata("openpipe-art").get_all("Provides-Extra") or ()
+          )
+          bundle = Path(art.__file__).with_name("_megatron_runtime")
+          assert (bundle / "manifest.json").is_file()
+          assert (bundle / "uv.lock").is_file()
+
+          from art.megatron.runtime.managed import _copy_art
+
+          with tempfile.TemporaryDirectory() as temp_dir:
+              managed = Path(temp_dir) / "managed"
+              venv.EnvBuilder(with_pip=False).create(managed)
+              managed_python = managed / "bin" / "python"
+              _copy_art(managed_python)
+              managed_site = Path(
+                  subprocess.check_output(
+                      [
+                          str(managed_python),
+                          "-c",
+                          "import sysconfig; print(sysconfig.get_paths()['purelib'])",
+                      ],
+                      text=True,
+                  ).strip()
+              )
+              copied_bundle = managed_site / "art" / "_megatron_runtime"
+              assert (copied_bundle / "manifest.json").is_file()
+              assert (copied_bundle / "nixl-de8115ca.tar.gz").is_file()
+              assert not (managed_site / "art" / "_vllm_runtime").exists()
+          PY
+
+          PYTHONPATH="$GITHUB_WORKSPACE/examples/multinode" timeout 150s \
+            .venv/bin/art-monarch local \
+            --program program:main \
+            --port 0 \
+            --startup-timeout 90
+          test ! -e "$ART_VLLM_RUNTIME_CACHE_DIR"

.github/workflows/prek.yml

@@ -13,8 +13,7 @@ env:
   CI_PYTHON_MM: "3.12"
   CI_UV_CACHE_RELEASE_TAG: "prek-uv-cache"
   CI_UV_CACHE_ASSET_PREFIX: "prek-uv-cache"
-  CI_APEX_PARALLEL_BUILD: "8"
-  CI_APEX_NVCC_THREADS: "1"
+  CI_BUILD_JOBS: "8"
   CI_UV_BUILD_SLOTS: "2"
   UV_CACHE_DIR: "/root/.cache/uv"
   UV_LINK_MODE: "copy"
@@ -36,11 +35,11 @@ jobs:
           fp="$(python3 scripts/ci/compute_uv_fingerprint.py \
             --pyproject pyproject.toml \
             --uv-lock uv.lock \
+            --megatron-pyproject megatron_runtime/pyproject.toml \
+            --megatron-uv-lock megatron_runtime/uv.lock \
             --base-image "${CI_BASE_IMAGE}" \
             --python-mm "${CI_PYTHON_MM}" \
-            --torch-cuda-arch-list "${TORCH_CUDA_ARCH_LIST}" \
-            --ci-apex-parallel-build "${CI_APEX_PARALLEL_BUILD}" \
-            --ci-apex-nvcc-threads "${CI_APEX_NVCC_THREADS}")"
+            --torch-cuda-arch-list "${TORCH_CUDA_ARCH_LIST}")"
           echo "fingerprint=${fp}" >> "${GITHUB_OUTPUT}"
           echo "Expected uv cache fingerprint: ${fp}"
 
@@ -187,14 +186,7 @@ jobs:
 
       - name: Install Megatron dependencies
         run: |
-          original_pyproject="$(mktemp)"
-          cp pyproject.toml "${original_pyproject}"
-          cleanup() {
-            mv "${original_pyproject}" pyproject.toml
-          }
-          trap cleanup EXIT
-
-          cudnn_path="${GITHUB_WORKSPACE}/.venv/lib/python${CI_PYTHON_MM}/site-packages/nvidia/cudnn"
+          cudnn_path="${GITHUB_WORKSPACE}/megatron_runtime/.venv/lib/python${CI_PYTHON_MM}/site-packages/nvidia/cudnn"
           export CUDNN_PATH="${cudnn_path}"
           export CUDNN_HOME="${cudnn_path}"
           export CUDNN_INCLUDE_PATH="${cudnn_path}/include"
@@ -203,16 +195,13 @@ jobs:
           export LIBRARY_PATH="${CUDNN_LIBRARY_PATH}${LIBRARY_PATH:+:${LIBRARY_PATH}}"
           export LD_LIBRARY_PATH="${CUDNN_LIBRARY_PATH}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
           export UV_CONCURRENT_BUILDS="${CI_UV_BUILD_SLOTS}"
-          export CMAKE_BUILD_PARALLEL_LEVEL="${CI_APEX_PARALLEL_BUILD}"
-          export MAX_JOBS="${CI_APEX_PARALLEL_BUILD}"
-          export NINJAFLAGS="-j${CI_APEX_PARALLEL_BUILD}"
-          python3 scripts/ci/apply_ci_uv_build_overrides.py \
-            --pyproject pyproject.toml \
-            --apex-parallel-build "${CI_APEX_PARALLEL_BUILD}" \
-            --apex-nvcc-threads "${CI_APEX_NVCC_THREADS}"
-          echo "CI uv build overrides: APEX_PARALLEL_BUILD=${CI_APEX_PARALLEL_BUILD}, NVCC_APPEND_FLAGS=--threads ${CI_APEX_NVCC_THREADS}, UV_CONCURRENT_BUILDS=${CI_UV_BUILD_SLOTS}"
+          export CMAKE_BUILD_PARALLEL_LEVEL="${CI_BUILD_JOBS}"
+          export MAX_JOBS="${CI_BUILD_JOBS}"
+          export NINJAFLAGS="-j${CI_BUILD_JOBS}"
           uv --version
-          uv sync --extra megatron --extra langgraph --extra plotting --group dev --frozen --python "${CI_PYTHON_MM}"
+          uv sync --extra langgraph --extra plotting --group dev --frozen --python "${CI_PYTHON_MM}"
+          uv sync --project megatron_runtime --extra cuda12 --group test --frozen --python "${CI_PYTHON_MM}"
+          uv pip install --python megatron_runtime/.venv/bin/python --no-deps --editable .
 
       - name: Run prek hooks (lint, format, typecheck, uv.lock)
         run: |
@@ -223,9 +212,10 @@ jobs:
 
       - name: Run Megatron lightweight tests
         run: |
-          uv run --no-sync python -c "import megatron.core.packed_seq_params"
-          uv run --no-sync pytest --nbval --current-env --tb=short \
+          megatron_runtime/.venv/bin/python -c "import megatron.core.packed_seq_params"
+          megatron_runtime/.venv/bin/python -m pytest --nbval --current-env --tb=short \
             tests/unit/test_megatron_reference_logprobs.py \
+            tests/unit/test_preprocessing_tokenize.py::test_gemma4_normalizes_json_tool_arguments_for_mapping_template \
             tests/unit/test_moe_routing_replay.py \
             tests/unit/test_moe_routing_real_path.py \
             tests/unit/test_pipeline_trainer_local_backend.py \

.github/workflows/release.yml

@@ -106,8 +106,10 @@ jobs:
                   str(runtime_python),
                   "-c",
                   "import torch, vllm; "
+                  "from art_vllm_runtime.fast_metrics import FastMetricsSharedWriter; "
                   "from art_vllm_runtime.policy_spans import "
                   "_patch_lora_update_coordinator; "
+                  "writer = FastMetricsSharedWriter(); writer.close(); "
                   "_patch_lora_update_coordinator(); "
                   "print('runtime compatibility ok')",
               ],
@@ -137,22 +139,22 @@ jobs:
           git tag v${{ needs.build-package.outputs.version }}
           git push origin v${{ needs.build-package.outputs.version }}
 
-      - name: Publish draft release
+      - name: Upload assets to draft release
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           if gh release view v${{ needs.build-package.outputs.version }} --json isDraft | jq -r '.isDraft' | grep -q true; then
-            gh release edit v${{ needs.build-package.outputs.version }} --draft=false
+            gh release upload v${{ needs.build-package.outputs.version }} dist/*
           else
             echo "::error::No draft release found for v${{ needs.build-package.outputs.version }}"
             exit 1
           fi
 
-      - name: Upload assets to release
+      - name: Publish release
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          gh release upload v${{ needs.build-package.outputs.version }} dist/*
+          gh release edit v${{ needs.build-package.outputs.version }} --draft=false
 
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -24,3 +24,4 @@ replays/
 !/src/art/wandb/**
 /src/art/wandb/__pycache__/
 scratch/
+/progress_log.md

.skyignore

@@ -2,6 +2,9 @@ __pycache__/
 .art/
 # .env
 .venv/
+.ruff_cache/
+.pytest_cache/
+scratch/
 grpo_trainer_lora_model/
 logs/
 shared_cache.db
@@ -13,5 +16,6 @@ dist/
 dev/art-e/data/
 replays/
 /trajectories/
+/progress_log.md
 .DS_Store
 # .local/

THIRD-PARTY-NOTICES

@@ -51,3 +51,24 @@ This project vendors a modified HybridEP runtime derived from DeepEP:
 The applicable license text is retained at:
 
 - src/art/megatron/_hybrid_ep/LICENSE
+
+Megatron release wheels also embed pinned source archives used only as headers
+when building ART's HybridEP extension:
+
+- NVIDIA NIXL 1.3.2 (commit de8115ca97d3f8fb63a4988e9b4d4a038b2e0f72),
+  Apache License 2.0, https://github.com/ai-dynamo/nixl
+- OpenUCX 1.21.0, BSD 3-Clause License,
+  https://github.com/openucx/ucx
+
+The complete license and notice files remain in their respective bundled source
+archives. ART links HybridEP against the matching NIXL and UCX libraries shipped
+by the official `nixl-cu12` or `nixl-cu13` wheel; it does not redistribute those
+libraries itself.
+
+On first cross-host HybridEP use, ART may download the checksum-pinned etcd
+3.5.33 executable from its official GitHub release. etcd is licensed under the
+Apache License 2.0: https://github.com/etcd-io/etcd
+
+The CUDA 12 managed Megatron environment installs NVIDIA Apex from its pinned
+25.09 source tag under the BSD 3-Clause License:
+https://github.com/NVIDIA/apex

dev/sft/sft-from-file.py

@@ -9,8 +9,10 @@
 
 
 async def main():
-    backend = MegatronBackend()
-
+    art.init_megatron_runtime_config(
+        topology=art.MegatronTopologyConfig(),
+        packed_sequence_length=4096,
+    )
     model_name = "run-" + "".join(
         random.choices("abcdefghijklmnopqrstuvwxyz0123456789", k=8)
     )
@@ -20,14 +22,14 @@ async def main():
         project="sft-from-file",
         base_model="Qwen/Qwen3.6-35B-A3B",
     )
-    await model.register(backend)
-
-    await train_sft_from_file(
-        model=model,
-        file_path="dev/sft/dataset.jsonl",
-        epochs=1,
-        peak_lr=2e-4,
-    )
+    async with MegatronBackend() as backend:
+        await model.register(backend)
+        await train_sft_from_file(
+            model=model,
+            file_path="dev/sft/dataset.jsonl",
+            epochs=1,
+            peak_lr=2e-4,
+        )
 
     print("Training complete!")