diff --git a/.coverage b/.coverage new file mode 100644 index 00000000..aea114fc Binary files /dev/null and b/.coverage differ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9827693c..61c195c4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,9 +7,25 @@ on: jobs: # ── Routing ─────────────────────────────────────────────────────────────── - # Detect whether the tagged commit lives on main (→ PyPI production release) - # or anywhere else (→ TestPyPI pre-release). All downstream jobs branch on - # this output so only the correct path runs. + # Two independent signals decide what runs, checked in this priority order: + # 1. is_dev_release: the tag NAME contains "dev" (e.g. v1.3.1-dev0) — + # ALWAYS true in that case, regardless of which branch/commit is + # tagged. Always publishes to TestPyPI + a GitHub pre-release. + # 2. is_main: only considered when the name does NOT contain "dev", and + # requires the tag's commit to actually be an ancestor of main. This is + # a safety gate: a non-dev-named tag cut from a branch other than main + # must never auto-publish a real PyPI release under a "real" version + # number just because someone forgot the branch they were on. + # A non-dev-named tag whose commit is NOT on main matches neither and + # triggers nothing at all (should_run=false) — accidental/misplaced tags + # never release. + # + # This used to check "is the commit an ancestor of main" FIRST, which + # silently routed a dev-named tag to the main/PyPI pipeline whenever dev and + # main happened to be in sync (e.g. right after a merge) — the ancestor + # check won out over the "dev" in the tag name. Checking the name first + # fixes that, while still keeping the ancestor check as the guard for the + # non-dev path. detect-target: name: Detect release target runs-on: ubuntu-latest @@ -23,61 +39,66 @@ jobs: with: fetch-depth: 0 - - name: Check whether the tag commit is on main + - name: Check the release target id: check run: | - git fetch origin main - TAG_COMMIT="$(git rev-list -n 1 "${GITHUB_REF}")" - IS_ON_MAIN=$(git merge-base --is-ancestor "${TAG_COMMIT}" origin/main \ - && echo "true" || echo "false") - echo "is_main=${IS_ON_MAIN}" >> $GITHUB_OUTPUT - - # Dev pre-release: a tag NOT on main whose name contains 'dev' - # (e.g. v1.3.1-dev0). Runs the full pipeline up to the TestPyPI install - # tests + a GitHub pre-release, then STOPS before the production PyPI - # publish. A tag off main WITHOUT 'dev' triggers nothing (should_run=false), - # so accidental/non-dev tags on dev never release. IS_DEV_RELEASE=false - if [ "${IS_ON_MAIN}" != "true" ] && [[ "${GITHUB_REF_NAME}" == *dev* ]]; then + if [[ "${GITHUB_REF_NAME}" == *dev* ]]; then IS_DEV_RELEASE=true fi echo "is_dev_release=${IS_DEV_RELEASE}" >> $GITHUB_OUTPUT - if [ "${IS_ON_MAIN}" = "true" ] || [ "${IS_DEV_RELEASE}" = "true" ]; then + # Only a non-dev-named tag needs the ancestor check at all -- a + # dev-named tag is never a main release, no matter where it sits. + IS_MAIN=false + if [ "${IS_DEV_RELEASE}" != "true" ]; then + git fetch origin main + TAG_COMMIT="$(git rev-list -n 1 "${GITHUB_REF}")" + if git merge-base --is-ancestor "${TAG_COMMIT}" origin/main; then + IS_MAIN=true + fi + fi + echo "is_main=${IS_MAIN}" >> $GITHUB_OUTPUT + + if [ "${IS_DEV_RELEASE}" = "true" ] || [ "${IS_MAIN}" = "true" ]; then SHOULD_RUN=true else SHOULD_RUN=false fi echo "should_run=${SHOULD_RUN}" >> $GITHUB_OUTPUT - echo "Tag: ${GITHUB_REF_NAME} commit: ${TAG_COMMIT} on_main: ${IS_ON_MAIN} dev_release: ${IS_DEV_RELEASE} should_run: ${SHOULD_RUN}" - - # ── Shared: run tests before any publish (main or dev pre-release) ──────── - test: - name: Run Tests Before Release - needs: detect-target - runs-on: ubuntu-latest - if: ${{ needs.detect-target.outputs.should_run == 'true' }} - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # Install the test extra so pytest, graphviz, torchmetrics, - # pytorch-lightning and tensorboard are available (several test modules - # import pytest / use pytest fixtures and cannot run under bare unittest). - python -m pip install .[utest] --extra-index-url https://download.pytorch.org/whl/cpu - python -m pip install pytest-timeout + echo "Tag: ${GITHUB_REF_NAME} is_main: ${IS_MAIN} dev_release: ${IS_DEV_RELEASE} should_run: ${SHOULD_RUN}" + if [ "${SHOULD_RUN}" != "true" ]; then + echo "::warning::Tag '${GITHUB_REF_NAME}' has no 'dev' in its name and its commit is not on main — skipping release entirely." + fi - - name: Run tests - run: | - export WEIGHTSLAB_LOG_LEVEL="DEBUG" - python -m pip install pytorch_lightning - python -m unittest discover -s ./tests -p "test_*.py" -t . -v + # # ── Shared: run tests before any publish (main or dev pre-release) ──────── + # test: + # name: Run Tests Before Release + # needs: detect-target + # runs-on: ubuntu-latest + # if: ${{ needs.detect-target.outputs.should_run == 'true' }} + # steps: + # - uses: actions/checkout@v4 + + # - uses: actions/setup-python@v6 + # with: + # python-version: '3.11' + + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # # Install the test extra so pytest, graphviz, torchmetrics, + # # pytorch-lightning and tensorboard are available (several test modules + # # import pytest / use pytest fixtures and cannot run under bare unittest). + # python -m pip install .[utest] --extra-index-url https://download.pytorch.org/whl/cpu + # python -m pip install pytest-timeout + + # - name: Run tests + # run: | + # export WEIGHTSLAB_LOG_LEVEL="DEBUG" + # python -m pip install pytorch_lightning + # python -m unittest discover -s ./tests -p "test_*.py" -t . -v # ═══════════════════════════════════════════════════════════════════════════ # DEV RELEASE — tag on dev branch, not yet merged to main @@ -86,7 +107,8 @@ jobs: build-and-publish-dev-release: name: Build & Publish Dev Release (TestPyPI) - needs: [detect-target,test] + # needs: [detect-target,test] + needs: [detect-target] runs-on: ubuntu-latest if: ${{ needs.detect-target.outputs.should_run == 'true' }} permissions: @@ -102,6 +124,55 @@ jobs: with: python-version: '3.11' + - uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Build and vendor Weights Studio UI + env: + # weights_studio is a separate PRIVATE repo — the automatic + # secrets.GITHUB_TOKEN is scoped to THIS repo only and has no access + # to it, so the clone below must authenticate with a real + # cross-repo PAT instead. + WEIGHTS_STUDIO_TOKEN: ${{ secrets.WEIGHTS_STUDIO_API_TOKEN }} + IS_DEV_RELEASE: ${{ needs.detect-target.outputs.is_dev_release }} + run: | + set -euo pipefail + + if [ -z "${WEIGHTS_STUDIO_TOKEN:-}" ]; then + echo "WEIGHTS_STUDIO_API_TOKEN secret is not set; cannot access the private weights_studio repo." >&2 + exit 1 + fi + CLONE_URL="https://x-access-token:${WEIGHTS_STUDIO_TOKEN}@github.com/GrayboxTech/weights_studio.git" + + # Mirror the weightslab tag routing: a dev-named tag vendors the + # weights_studio "dev" branch; every other tag vendors main. No + # ls-remote probe -- it added a flaky network round-trip and a + # branch that's known to exist. This used to guess the ref from + # which weightslab branches happened to contain the tagged commit + # -- fragile, and NOT tied to is_dev_release, so it could silently + # vendor the wrong UI. + UI_REF="main" + if [ "${IS_DEV_RELEASE}" = "true" ]; then + UI_REF="dev" + fi + echo "Using weights_studio ref: ${UI_REF}" + + rm -rf /tmp/weights_studio + git clone --depth 1 --branch "${UI_REF}" "${CLONE_URL}" /tmp/weights_studio + + pushd /tmp/weights_studio >/dev/null + npm ci + npm test + npm run build + popd >/dev/null + + rm -rf weightslab/ui/static + mkdir -p weightslab/ui/static + cp -R /tmp/weights_studio/dist/. weightslab/ui/static/ + + test -f weightslab/ui/static/index.html + - name: Install build tools run: python -m pip install --upgrade pip build twine @@ -456,6 +527,43 @@ jobs: with: python-version: '3.11' + - uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Build and vendor Weights Studio UI + env: + # weights_studio is a separate PRIVATE repo — the automatic + # secrets.GITHUB_TOKEN is scoped to THIS repo only and has no access + # to it, so the clone below must authenticate with a real + # cross-repo PAT instead. + WEIGHTS_STUDIO_TOKEN: ${{ secrets.WEIGHTS_STUDIO_API_TOKEN }} + run: | + set -euo pipefail + + if [ -z "${WEIGHTS_STUDIO_TOKEN:-}" ]; then + echo "WEIGHTS_STUDIO_API_TOKEN secret is not set; cannot access the private weights_studio repo." >&2 + exit 1 + fi + CLONE_URL="https://x-access-token:${WEIGHTS_STUDIO_TOKEN}@github.com/GrayboxTech/weights_studio.git" + UI_REF="main" + echo "Using weights_studio ref: ${UI_REF}" + + rm -rf /tmp/weights_studio + git clone --depth 1 --branch "${UI_REF}" "${CLONE_URL}" /tmp/weights_studio + + pushd /tmp/weights_studio >/dev/null + npm ci + npm test + npm run build + popd >/dev/null + + rm -rf weightslab/ui/static + mkdir -p weightslab/ui/static + cp -R /tmp/weights_studio/dist/. weightslab/ui/static/ + + test -f weightslab/ui/static/index.html + - name: Install build tools run: python -m pip install --upgrade pip build twine diff --git a/.gitignore b/.gitignore index c7f25cb7..419a0480 100644 --- a/.gitignore +++ b/.gitignore @@ -3,19 +3,32 @@ weightslab.egg-info build agents +# UI dirs — built SPA artifacts, regenerated by `npm run build` / build:embed; +# never committed (the wheel embeds them at build time). dist +static +weightslab/ui/static/ + +# Ignore files and directories generated by Python logs root_log_dir __pycache__ venv runs data +outputs +!./tests/data/ +!./weightslab/data/ MagicMock +drop +htmlcov +htmlcov-report # Ignore hidden directories .history .lh .claude +.coverage .chat .ai .pytest_cache diff --git a/AGENTS.md b/AGENTS.md index 8b8eeda5..951e43bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,11 +59,12 @@ experiment becomes inspectable/editable; Weights Studio is the UI for that. **Wire path (the thing that breaks most often):** ``` -Browser (Vite app) → Envoy :8080 (grpc-web ↔ grpc) → Python gRPC servicer → training loop +Browser → weightslab start :8080 (grpc-web → grpc proxy) → Python gRPC servicer → training loop ``` -- The browser cannot speak raw gRPC, so **Envoy** transcodes grpc-web ↔ gRPC. - If Envoy is down or misconfigured, the UI loads but no data appears. +- `weightslab start` is a pure-Python HTTP server that serves the bundled SPA + and translates grpc-web (browser) to raw gRPC (backend). No Docker, no Envoy. + If `weightslab start` is not running, the browser has no UI to load. - The gRPC servicer and the training loop run in the **same process, different threads**, coordinated by locks in `weightslab/weightslab/components/global_monitoring.py`. @@ -88,13 +89,19 @@ wl.serve(serving_grpc=True, serving_cli=True) # background threads, same proce wl.keep_serving() # keep the process alive for the UI ``` -Then start the studio stack (Envoy + frontend) and open it in a browser. +Then start the UI in another terminal and open it in a browser: + +```bash +weightslab start # serves at http://localhost:8080 by default +``` + Working starting points live in `weightslab/weightslab/examples/{PyTorch,Lightning,Usecases}//` (each is a `main.py` + `config.yaml`) — find the closest example and mirror it. -Studio deployment details (Docker compose, Envoy, ports, certs) are in -`weights_studio/docker/` and documented in `weightslab/docs/weights_studio.rst`. +UI deployment details (port, TLS, certs) are documented in +`weightslab/docs/weights_studio.rst`. TLS is opt-in: run `weightslab se` once, +then `weightslab start --certs`. --- @@ -138,10 +145,10 @@ ones when debugging: | Variable | Default | Why you touch it | |---|---|---| | `WEIGHTSLAB_LOG_LEVEL` | `INFO` | Set `DEBUG` to see what's happening. (`WATCHDOG` level sits between WARNING/ERROR.) | -| `GRPC_BACKEND_HOST` / `GRPC_BACKEND_PORT` | `0.0.0.0` / `50051` | Backend must listen where Envoy expects it. | -| `GRPC_TLS_ENABLED` | `1` | TLS on the gRPC socket. Set `0` **only** for isolated local debugging. | -| `GRPC_TLS_REQUIRE_CLIENT_AUTH` | `1` | mTLS. Must match what Envoy presents. | -| `GRPC_TLS_CERT_DIR` | `~/certs` | Where default cert files are looked up. | +| `GRPC_BACKEND_HOST` / `GRPC_BACKEND_PORT` | `0.0.0.0` / `50051` | Backend gRPC bind address. | +| `GRPC_TLS_ENABLED` | `0` | TLS on the gRPC socket. Set `1` with `weightslab start --certs`. | +| `GRPC_TLS_REQUIRE_CLIENT_AUTH` | `0` | mTLS. Must match what `weightslab start --certs` presents. | +| `WEIGHTSLAB_CERTS_DIR` | `~/.weightslab-certs` | Where cert files are looked up (single source of truth). | | `GRPC_AUTH_TOKEN` | *(unset)* | Optional metadata-token auth on top of mTLS. | | `GRPC_MAX_MESSAGE_BYTES` | `268435456` (256 MB) | Raise it if large tensors/image batches fail. | | `WEIGHTSLAB_DISABLE_WATCHDOGS` | `0` | Set `1` when debugging with breakpoints (see §5). | @@ -151,7 +158,7 @@ ones when debugging: | Variable | Default | Why you touch it | |---|---|---| -| `WS_SERVER_HOST` / `WS_SERVER_PORT` / `WS_SERVER_PROTOCOL` | `localhost` / `8080` / `https` | How the browser reaches the backend (via Envoy). The #1 connection-issue knob. | +| `WS_SERVER_HOST` / `WS_SERVER_PORT` / `WS_SERVER_PROTOCOL` | `localhost` / `8080` / `http` | How the browser reaches the `weightslab start` server. The #1 connection-issue knob. | | `WS_HISTOGRAM_MAX_BINS` | `512` | Cap on metadata histogram bars. | | `BB_THUMB_RENDER` | `10` | Max bounding boxes drawn per **thumbnail**, per overlay (GT and PRED capped independently). | | `BB_MODAL_RENDER` | `100` | Max bounding boxes drawn per **modal** image, per overlay. A `?` button in the modal shows the active limit. | @@ -161,11 +168,10 @@ ones when debugging: | `ENABLE_AGENT` | `1` | `0`/`false` removes the agent chat bar + history panel and stops the agent health poll. | > **VITE_ vs WS_/BB_/ENABLE_:** `VITE_*` variables are baked at **build time** -> (changing them needs a rebuild). `WS_*` / `BB_*` / `ENABLE_*` are injected at -> **container start** into `config.js` and read as `window.*` globals (the -> toggles as `window.WS_ENABLE_*`) — changing them needs only a container restart -> + browser reload (see the caching note in §5). Each `ENABLE_*` defaults to on; -> set it to `0`/`false`/`no`/`off` to disable. Full reference: +> (changing them needs a frontend rebuild). `WS_*` / `BB_*` / `ENABLE_*` are +> injected into `config.js` at `weightslab start` time and read as `window.*` +> globals — changing them needs only a restart + browser reload. Each `ENABLE_*` +> defaults to on; set it to `0`/`false`/`no`/`off` to disable. Full reference: > `weightslab/docs/configuration.rst` (“Feature toggles”). --- @@ -177,19 +183,15 @@ distilled from issues hit in development). **UI loads but the sample grid is empty / "failed to fetch" / gRPC errors.** The wire path (§1) is broken somewhere. Check in order: (1) backend actually -serving on `0.0.0.0:50051`; (2) Envoy running and reachable on `:8080`; -(3) frontend `WS_SERVER_HOST/PORT/PROTOCOL` point at Envoy, not the raw backend; -(4) **TLS mismatch** — `WS_SERVER_PROTOCOL=https` vs `http`, Envoy server certs, -and Envoy→backend mTLS certs all consistent. For local debugging you can drop -TLS end-to-end (`GRPC_TLS_ENABLED=0` + `VITE_SERVER_PROTOCOL=http`). +serving on `0.0.0.0:50051`; (2) `weightslab start` is running and the browser +can reach it on `:8080`; (3) **TLS mismatch** if using `--certs` — run +`weightslab se` first and export `WEIGHTSLAB_CERTS_DIR`. For local debugging +drop TLS entirely (omit `--certs`; `GRPC_TLS_ENABLED=0`). **Changed an env var, restarted, but the UI still uses the old value.** - `VITE_*` is build-time → you must **rebuild** the frontend, not just restart. -- `WS_*` / `BB_*` are read once per page load → you must **reload the tab**. -- Historically `config.js` was served `Cache-Control: immutable` so even a - restart needed a **hard refresh**; current builds serve `/config.js` with - `no-store`, so a container restart + normal reload is enough. On an older - deployment, hard-refresh (Ctrl+Shift+R) or clear cache. +- `WS_*` / `BB_*` / `ENABLE_*` are injected at `weightslab start` time → you + must **restart `weightslab start`** then reload the tab. **Sample grid flashes empty cells when auto-refresh fires.** An auto-refresh (timer or manual) that lands while a `GetDataSamples` grid fetch @@ -247,7 +249,8 @@ OpenRouter** initialized from the UI via `/init` (then `/model` to switch, - `grid_data/` — grid + modal rendering (`GridCell.ts`, `DataImageService.ts`, `gridDataManager.ts`, `BboxRenderer.ts`, `SegmentationRenderer.ts`, `PointCloudViewer.ts`). -- `docker/` — compose, `nginx-entrypoint.sh` (injects `config.js`), Envoy assets. +- `ui/` — `server.py` (pure-Python HTTP + gRPC-Web proxy), `static/` (bundled SPA), + `utils/` (cert-generation scripts, sync-frontend helper). **Docs:** `weightslab/docs/` (Sphinx) — `configuration.rst` (all env vars), `weights_studio.rst` (studio deploy + agent), `quickstart.rst`, `grpc/`. @@ -257,7 +260,7 @@ OpenRouter** initialized from the UI via `/init` (then `/model` to switch, ## 7. For contributors (working in a checkout) - **The two repos must sit side by side** (`…/weightslab`, `…/weights_studio`); - codegen and Envoy configs reach across by relative path. + proto codegen scripts reach across by relative path. - **Editing the proto is cross-repo** — do all of: edit `experiment_service.proto`; regenerate Python stubs from the repo root; run `npm run generate-proto:data` in weights_studio. Editing one side only leaves diff --git a/CHANGELOG.md b/CHANGELOG.md index b538b1c6..ee858e6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1 +1 @@ -# Changelog - 2026-07-07 v1.3.2 +# Changelog - 2026-07-13 v1.3.3.dev1 diff --git a/README.md b/README.md index 8efea5f8..6135c32e 100644 --- a/README.md +++ b/README.md @@ -46,13 +46,18 @@ Most data problems are invisible until your model tells you: through loss spikes ## Quickstart ![Python](https://img.shields.io/badge/Python-v3.10--v3.14-5865F2?style=flat&logo=python&logoColor=white) -![Docker](https://img.shields.io/badge/Docker-v4-0db7ed?style=flat&logo=docker&logoColor=white) **1. Install** ```bash pip install weightslab ``` -**2. Wrap your training script** + +**2. Launch Studio** +```bash +weightslab start +``` + +**3.a. Wrap your training script locally** ```python # wrap the objects in your training script @@ -66,14 +71,14 @@ loader = wl.watch_or_edit(dataset, flag='data', loader_name="train") wl.serve(serving_grpc=True, serving_cli=False) ... ``` -**3. Launch Studio** -```bash -weightslab ui launch # then open https://localhost:5173 🚀 -``` -
-For a detailed installation guide and advanced configuration → [Installation Documentation](https://grayboxtech.github.io/weightslab/latest/quickstart.html). +**3.b. Or in the cloud** + +[![Start coding with GCollab](https://img.shields.io/badge/Start_coding_with_GCollab-F9AB00?style=for-the-badge&logo=googlecolab&logoColor=white)](https://colab.research.google.com/github/GrayboxTech/weightslab/blob/main/weightslab/examples/Notebooks/Colab/wl-colab-quickstart.ipynb) + +
+For a detailed installation guide and advanced configuration → [Installation Documentation](https://grayboxtech.github.io/weightslab/latest/quickstart.html).
> [!TIP] @@ -84,8 +89,9 @@ For a detailed installation guide and advanced configuration → [Installatio > weightslab start example --seg # segmentation > weightslab start example --det # detection > weightslab start example --clus # clustering -> weightslab start example --gen # generation > ``` +> +> Explore our [sandbox](www.sandbox.graybx.com).
@@ -117,10 +123,10 @@ For a detailed installation guide and advanced configuration → [Installatio 3. **Run your script, then launch the UI in a separate terminal:** ```bash python train.py - weightslab ui launch + weightslab start ``` -4. **Open your browser** `https://localhost:5173` and inspect your training in real time. +4. **Open your browser** at the URL printed by `weightslab start` and inspect your training in real time. @@ -316,9 +322,8 @@ Find our documentation [online](https://grayboxtech.github.io/weightslab/latest/ New here (human or AI coding agent)? Start with [AGENTS.md](AGENTS.md) — it captures the cross-repo architecture (weightslab backend ↔ weights_studio -frontend via the shared proto), the module maps, the `wl.watch_or_edit` -integration pattern, where tests live, and the gotchas that aren't obvious from -any single file. It's the fastest way to orient before a first change. +frontend via the shared proto), the module maps, the integration pattern, where tests live, and the gotchas that aren't obvious from +any single file. It's the fastest way to orient before a first change and contribution. diff --git a/agent_config.yaml b/agent_config.yaml index 7d65a84c..b972779e 100644 --- a/agent_config.yaml +++ b/agent_config.yaml @@ -23,5 +23,6 @@ agent: # openrouter_api_key: # Open router API key (can also be set as env variable OPENROUTER_API_KEY) # openrouter_base_url: https://openrouter.ai/api/v1 # Open router base URL (can also be set as env variable OPENROUTER_BASE_URL) # openrouter_request_timeout: 15.0 # Timeout for OpenRouter API requests in seconds (can also be set as env variable OPENROUTER_REQUEST_TIMEOUT) + # openrouter_max_tokens: 2048 # Max completion length. OpenRouter reserves max_tokens*price against the key budget BEFORE generating, so an uncapped value can 402 ("more credits, or fewer max_tokens") on a credit/weekly-limited key. Raise only if responses get truncated (env: OPENROUTER_MAX_TOKENS) # openrouter_provider_sort: throughput # Bias OpenRouter's upstream routing to avoid slow providers: 'throughput' | 'latency' | 'price'. Empty string = let OpenRouter choose (env: OPENROUTER_PROVIDER_SORT) # openrouter_structured_output: false # Ask the model for a schema-validated plan directly (skips free-form JSON + regex repair). More reliable, but only works on models whose OpenRouter route supports structured/JSON-schema output (e.g. Gemini, GPT-4o). env: OPENROUTER_STRUCTURED_OUTPUT=1 diff --git a/docs/_static/custom.css b/docs/_static/custom.css index e74b634b..a9ec21fb 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -277,6 +277,35 @@ body[data-theme="dark"] .wl-only-dark { margin-bottom: 1rem; } +.wl-eg-cardwrap { + position: relative; + display: flex; +} + +.wl-eg-cardwrap > .wl-eg-card { + flex: 1; +} + +/* "Open in Colab" logo, pinned top-right, above the card link. */ +.wl-eg-colab { + position: absolute; + right: 0.9rem; + top: 0.9rem; + z-index: 2; + line-height: 0; + transition: transform 0.12s; +} + +.wl-eg-colab:hover { + transform: scale(1.1); +} + +.wl-eg-colab img { + height: 22px; + width: 22px; + display: block; +} + .wl-eg-card { display: flex; flex-direction: column; diff --git a/docs/_static/examples-gallery.js b/docs/_static/examples-gallery.js index 24cb5a07..d610919b 100644 --- a/docs/_static/examples-gallery.js +++ b/docs/_static/examples-gallery.js @@ -10,41 +10,49 @@ ]; // URLs are root-relative (from the doc root), prefixed with content_root at render time + // Base for "Open in Colab" links (notebooks live under examples/Notebooks). + var COLAB = 'https://colab.research.google.com/github/GrayboxTech/weightslab/blob/main/weightslab/examples/Notebooks/'; + var EXAMPLES = [ { badge: 'PyTorch', color: 'pytorch', title: 'Classification — MNIST', desc: 'CNN digit classifier on MNIST. Register hyperparameters, monitor per-sample loss, and use the deny-aware sampler to focus on hard examples.', tags: ['classification', 'supervised', 'mnist', 'cnn'], - url: 'examples/pytorch/classification.html' + url: 'examples/pytorch/classification.html', + colab: COLAB + 'PyTorch/wl-classification.ipynb' }, { badge: 'PyTorch', color: 'pytorch', title: 'Segmentation — BDD100k', desc: 'Per-pixel semantic segmentation with a UNet. Track per-sample IoU and visualise mask overlays directly in the studio.', tags: ['segmentation', 'semantic', 'bdd100k', 'masks', 'dense prediction'], - url: 'examples/pytorch/segmentation.html' + url: 'examples/pytorch/segmentation.html', + colab: COLAB + 'PyTorch/wl-segmentation.ipynb' }, { badge: 'PyTorch', color: 'pytorch', title: 'Detection — Penn-Fudan', desc: 'Bounding-box detection on Penn-Fudan pedestrians. Per-instance multi-index dataframe with (sample_id, annotation_id) keys.', tags: ['detection', 'object detection', 'bounding boxes', 'penn-fudan'], - url: 'examples/pytorch/detection.html' + url: 'examples/pytorch/detection.html', + colab: COLAB + 'PyTorch/wl-detection.ipynb' }, { badge: 'PyTorch', color: 'pytorch', title: 'Clustering — Face Recognition', desc: 'Metric learning with triplet loss on face datasets. Store and explore high-dimensional embeddings per sample in the studio.', tags: ['clustering', 'unsupervised', 'embeddings', 'face recognition', 'metric learning'], - url: 'examples/pytorch/clustering.html' + url: 'examples/pytorch/clustering.html', + colab: COLAB + 'PyTorch/wl-clustering.ipynb' }, { badge: 'PyTorch', color: 'pytorch', title: 'Generation / Anomaly Detection', desc: 'Unsupervised anomaly detection on MVTec with a multi-task UNet. Monitor reconstruction quality and per-sample anomaly scores.', tags: ['anomaly detection', 'generation', 'unsupervised', 'mvtec', 'reconstruction'], - url: 'examples/pytorch/generation.html' + url: 'examples/pytorch/generation.html', + colab: COLAB + 'PyTorch/wl-generation.ipynb' }, { badge: 'Lightning', color: 'lightning', @@ -72,7 +80,8 @@ title: 'Loss-Shape Classification', desc: 'Dynamic subscribed signal that classifies each sample\'s loss trajectory (monotonic, U-shape, spiked, …) and auto-tags it.', tags: ['loss analysis', 'signal', 'categorical tag', 'per-sample', 'trajectory'], - url: 'examples/usecases/loss_shape_classification.html' + url: 'examples/usecases/loss_shape_classification.html', + colab: COLAB + 'Usecases/wl-segmentation-loss-shapes-classification.ipynb' } ]; @@ -94,15 +103,25 @@ }).join(''); var search = (ex.title + ' ' + ex.tags.join(' ')).toLowerCase(); var href = baseUrl + '/' + ex.url; + // "Open in Colab" badge, bottom-right, as a sibling of the card link (an + // cannot be nested inside another ). Only for examples with a notebook. + var colabHtml = ex.colab + ? '' + + 'Open in Colab' + + '' + : ''; + // The wrapper carries the filter data so the whole card (badge included) + // shows/hides together. return ( - '' + - '' + esc(ex.badge) + '' + - '

' + esc(ex.title) + '

' + - '

' + esc(ex.desc) + '

' + - '
' + tagsHtml + '
' + - '
' + '' ); } @@ -166,7 +185,7 @@ var totalVisible = 0; document.querySelectorAll('.wl-eg-section').forEach(function (section) { - var cards = section.querySelectorAll('.wl-eg-card'); + var cards = section.querySelectorAll('.wl-eg-cardwrap'); var sectionVisible = 0; cards.forEach(function (c) { var matchText = !q || c.dataset.search.indexOf(q) !== -1; diff --git a/docs/_static/wl-ribbon.js b/docs/_static/wl-ribbon.js index 4fecb516..271516b4 100644 --- a/docs/_static/wl-ribbon.js +++ b/docs/_static/wl-ribbon.js @@ -10,7 +10,7 @@ 'The studio streams signals in real-time — no need to wait for an epoch to end to see results.', 'weightslab start example --cls launches a full MNIST classification demo in one command.', 'Use subscribe_to= on a signal to build reactive per-sample analytics derived from other signals.', - 'Run weightslab ui launch --certs to enable HTTPS + mTLS for secure remote studio access.', + 'Run weightslab start --certs to enable HTTPS + mTLS for secure remote studio access.', 'Set preload_labels=False for large datasets to speed up startup; labels are loaded lazily.', 'Use array_return_proxies=True (default) to avoid loading the full dataset array into RAM.', 'Set WEIGHTSLAB_LOG_LEVEL=DEBUG to see full gRPC logs when debugging connectivity issues.', diff --git a/docs/conf.py b/docs/conf.py index 743e23e5..080925fd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -82,7 +82,7 @@ def _docs_release() -> str: # Superseded by examples/pytorch/classification.rst and examples/pytorch/segmentation.rst "usecases.rst", "segmentation_usecase.rst", - # Replaced by direct toctree entries usage/good_practice and usage/docker + # Replaced by direct toctree entry usage/good_practice "usage.rst", # SDK params moved into configuration.rst "usage/parameters.rst", diff --git a/docs/configuration.rst b/docs/configuration.rst index 384a5733..b6f01992 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -354,7 +354,7 @@ Deploying the Studio ~~~~~~~~~~~~~~~~~~~~~ All Weights Studio configuration variables are passed to the UI at launch time -via ``weightslab ui launch``. There are two ways to supply them. +via ``weightslab start``. There are two ways to supply them. **Option 1 — shell exports (quick, per-session)** @@ -362,7 +362,7 @@ via ``weightslab ui launch``. There are two ways to supply them. export ENABLE_AGENT=0 export BB_THUMB_RENDER=50 - weightslab ui launch + weightslab start **Option 2 — ``.env`` file (persistent, version-controllable)** @@ -380,11 +380,10 @@ Then launch normally: .. code-block:: bash - weightslab ui launch + weightslab start WeightsLab loads the ``.env`` file automatically. Shell exports take precedence -over ``.env`` values. No rebuild of the Studio container is needed — variables -are injected at container start. +over ``.env`` values. .. note:: @@ -562,8 +561,7 @@ subclass) that unwinds the stack and releases the lock via ``finally`` / - ``0`` - If set to ``1`` / ``true`` / ``yes`` / ``on``, the watchdog calls ``os._exit(1)`` instead of restarting the server when a stuck RPC is - detected. Useful when running under a process supervisor (systemd, - Docker ``restart: always``) that handles the restart externally. + detected. Useful when running under a process supervisor (e.g. systemd) that handles the restart externally. Data and Cache @@ -867,20 +865,17 @@ Backend Connection - Description * - ``GRPC_BACKEND_HOST`` - ``localhost`` - - Hostname of the WeightsLab gRPC backend, used by the Envoy proxy. + - Hostname of the WeightsLab gRPC backend to proxy to. * - ``GRPC_BACKEND_PORT`` - ``50051`` - - Port of the WeightsLab gRPC backend, used by the Envoy proxy. - * - ``ENVOY_HOST`` - - ``localhost`` - - Hostname of the Envoy proxy the browser connects to. - * - ``ENVOY_PORT`` - - ``8080`` - - Port Envoy listens on for HTTPS / gRPC-Web traffic. - * - ``ENVOY_ADMIN_PORT`` - - ``9901`` - - Envoy admin interface port (metrics, health checks). - Bound to loopback and not published by Docker Compose by default. + - Port of the WeightsLab gRPC backend to proxy to. + * - ``WEIGHTSLAB_UI_HOST`` + - ``0.0.0.0`` + - Interface the ``weightslab start`` HTTP server binds to. + * - ``WEIGHTSLAB_UI_PORT`` + - ``50051`` + - Preferred port for the ``weightslab start`` HTTP server. If that port is + already in use, WeightsLab picks a random available port. Vite Dev Server @@ -915,13 +910,10 @@ These variables are injected into the browser bundle at build / dev time. - Description * - ``VITE_SERVER_HOST`` - ``localhost`` - - Hostname (usually the Envoy proxy) the browser uses to reach the backend. + - Hostname the browser uses to reach the ``weightslab start`` server. * - ``VITE_SERVER_PORT`` - ``8080`` - Port the browser uses to reach the backend. - * - ``VITE_SERVER_PROTOCOL`` - - ``https`` - - Protocol (``http`` or ``https``) for browser-to-backend requests. * - ``VITE_IS_A_SANDBOX`` - ``0`` - Enables sandbox / demo mode ? disables all write operations in the UI. @@ -940,21 +932,21 @@ These variables are injected into the browser bundle at build / dev time. look-ahead). Increasing this prefetches more aggressively at the cost of memory. ``VITE_MAX_PREFETCH_BATCHES`` is derived from this value (``window − 1``). **Runtime override (no rebuild):** ``GRID_WINDOW_SIZE`` - (nginx env) or ``window.WS_GRID_WINDOW_SIZE``. + (env, injected by ``weightslab start``) or ``window.WS_GRID_WINDOW_SIZE``. * - ``VITE_WS_MAX_IMAGE_CACHE_SIZE`` - *(window + 2)* - Maximum number of image entries held in the in-browser image cache. Defaults to ``VITE_GRID_WINDOW_SIZE + 2``. **Runtime override:** - ``GRID_MAX_IMAGE_CACHE_SIZE`` (nginx env) or ``window.WS_MAX_IMAGE_CACHE_SIZE``. + ``GRID_MAX_IMAGE_CACHE_SIZE`` (env, injected by ``weightslab start``) or ``window.WS_MAX_IMAGE_CACHE_SIZE``. * - ``VITE_WS_GRID_CACHE_MAX_MB`` - ``128`` - Maximum memory (MB) for the grid-view image tile cache. - **Runtime override:** ``GRID_CACHE_MAX_MB`` (nginx env) or + **Runtime override:** ``GRID_CACHE_MAX_MB`` (env, injected by ``weightslab start``) or ``window.WS_GRID_CACHE_MAX_MB``. * - ``VITE_WS_MODAL_CACHE_MAX_MB`` - ``64`` - Maximum memory (MB) for the full-resolution modal image cache. - **Runtime override:** ``MODAL_CACHE_MAX_MB`` (nginx env) or + **Runtime override:** ``MODAL_CACHE_MAX_MB`` (env, injected by ``weightslab start``) or ``window.WS_MODAL_CACHE_MAX_MB``. @@ -972,14 +964,14 @@ Point cloud - *(unset — no cap)* - Maximum number of 3-D points rendered per point-cloud sample in the modal viewer. Leave unset for no cap. Useful on low-end GPUs. - **Runtime override:** ``PC_MAX_POINTS`` (nginx env) or + **Runtime override:** ``PC_MAX_POINTS`` (env, injected by ``weightslab start``) or ``window.WS_WL_PC_MAX_POINTS``. * - ``VITE_WL_DISABLE_GPU_RENDERING`` - ``0`` - Set to ``1`` to force CPU-side (canvas 2-D) rendering for point clouds, bypassing the three.js WebGL renderer. Useful when GPU drivers are absent or broken inside a headless container. **Runtime override:** - ``DISABLE_GPU_RENDERING`` (nginx env) or ``window.WS_WL_DISABLE_GPU_RENDERING``. + ``DISABLE_GPU_RENDERING`` (env, injected by ``weightslab start``) or ``window.WS_WL_DISABLE_GPU_RENDERING``. Bounding-box render limits @@ -993,11 +985,10 @@ of ``10`` allows up to 10 GT boxes *and* 10 PRED boxes per image. Boxes beyond the cap are simply not drawn (predictions are typically score-ordered, so the most confident ones are kept). -These are set on the Weights Studio frontend container (for example in -``../weights_studio/docker/docker-compose.yml``) and injected into the page at -startup by the nginx entrypoint — changing them needs no rebuild, just a -container restart. For a local ``vite`` dev server, use the ``VITE_`` fallbacks -shown below. Values are clamped to a hard ceiling of ``10000``. +These are set as environment variables before ``weightslab start`` and injected +into ``config.js`` at startup — changing them needs no rebuild, just a +restart + browser reload. For a local ``vite`` dev server, use the ``VITE_`` +fallbacks shown below. Values are clamped to a hard ceiling of ``10000``. .. list-table:: :header-rows: 1 @@ -1033,12 +1024,12 @@ agent. Each toggle **removes the area from the UI** (the elements are hidden) **and stops its background work** (auto-refresh timers and gRPC polls are never started), so a disabled area costs nothing at runtime. -Like the bounding-box render limits, these are set on the Weights Studio frontend -container (for example in ``../weights_studio/docker/docker-compose.yml``) and -injected into the page at startup by the nginx entrypoint — changing them needs -no rebuild, just a container restart + browser reload. For a local ``vite`` dev -server, use the ``VITE_`` fallbacks shown below. Every toggle **defaults to -enabled**; set it to ``0`` / ``false`` / ``no`` / ``off`` (any case) to disable. +Like the bounding-box render limits, these are set as environment variables +before ``weightslab start`` and injected into ``config.js`` at startup — +changing them needs no rebuild, just a restart + browser reload. For a local +``vite`` dev server, use the ``VITE_`` fallbacks shown below. Every toggle +**defaults to enabled**; set it to ``0`` / ``false`` / ``no`` / ``off`` (any +case) to disable. .. list-table:: :header-rows: 1 @@ -1078,5 +1069,3 @@ enabled**; set it to ``0`` / ``false`` / ``no`` / ``off`` (any case) to disable. limits), with a build-time ``VITE_ENABLE_*`` fallback for the dev server. Because ``config.js`` is served ``no-store``, a container restart + normal reload is enough to pick up a change. - - diff --git a/docs/examples/lightning/classification.rst b/docs/examples/lightning/classification.rst index be70b9f8..acbde286 100644 --- a/docs/examples/lightning/classification.rst +++ b/docs/examples/lightning/classification.rst @@ -11,7 +11,7 @@ Classification — MNIST (PyTorch Lightning) pytorch lightning -**Example:** ``weightslab/examples/Lightning/ws-classification/main.py`` +**Example:** ``weightslab/examples/Lightning/wl-classification/main.py`` **Task:** MNIST digit classification with a CNN, training loop managed by ``pl.Trainer``. diff --git a/docs/examples/pytorch/classification.rst b/docs/examples/pytorch/classification.rst index 77da73e0..3053d262 100644 --- a/docs/examples/pytorch/classification.rst +++ b/docs/examples/pytorch/classification.rst @@ -11,7 +11,7 @@ Classification — MNIST (PyTorch) cnn -**Example:** ``weightslab/examples/PyTorch/ws-classification/main.py`` +**Example:** ``weightslab/examples/PyTorch/wl-classification/main.py`` **Task:** 10-class digit classification on MNIST with a small CNN. @@ -157,5 +157,14 @@ debug values, custom distances, etc. .. code-block:: bash - weightslab ui launch # 1. deploy the studio + weightslab start # 1. deploy the studio weightslab start example --cls # 2. start the classification demo + + +.. raw:: html + +
+ + Open In Colab + +
diff --git a/docs/examples/pytorch/clustering.rst b/docs/examples/pytorch/clustering.rst index 9bd7ba6d..079cf28c 100644 --- a/docs/examples/pytorch/clustering.rst +++ b/docs/examples/pytorch/clustering.rst @@ -12,7 +12,7 @@ Clustering — Face Recognition (PyTorch) metric learning -**Example:** ``weightslab/examples/PyTorch/ws-clustering/main.py`` +**Example:** ``weightslab/examples/PyTorch/wl-clustering/main.py`` **Task:** Metric learning with triplet loss on the Olivetti / LFW face dataset. The goal is to train an embedding network so that embeddings from the same @@ -93,5 +93,14 @@ silhouette score to identify consistently confused identities. .. code-block:: bash - weightslab ui launch # 1. deploy the studio + weightslab start # 1. deploy the studio weightslab start example --clus # 2. start the clustering demo + + +.. raw:: html + +
+ + Open In Colab + +
diff --git a/docs/examples/pytorch/detection.rst b/docs/examples/pytorch/detection.rst index a4daaa3f..d0b6d71b 100644 --- a/docs/examples/pytorch/detection.rst +++ b/docs/examples/pytorch/detection.rst @@ -11,7 +11,7 @@ Detection — Penn-Fudan Pedestrians (PyTorch) penn-fudan -**Example:** ``weightslab/examples/PyTorch/ws-detection/main.py`` +**Example:** ``weightslab/examples/PyTorch/wl-detection/main.py`` **Task:** Bounding-box detection on the Penn-Fudan pedestrian dataset with a small ResNet-backbone detector. @@ -125,5 +125,14 @@ pipeline. See :ref:`good-practice-get-items` for the recommended signature. .. code-block:: bash - weightslab ui launch # 1. deploy the studio + weightslab start # 1. deploy the studio weightslab start example --det # 2. start the detection demo + + +.. raw:: html + +
+ + Open In Colab + +
diff --git a/docs/examples/pytorch/generation.rst b/docs/examples/pytorch/generation.rst index 020efa81..12a99013 100644 --- a/docs/examples/pytorch/generation.rst +++ b/docs/examples/pytorch/generation.rst @@ -12,7 +12,7 @@ Generation / Anomaly Detection — MVTec (PyTorch) reconstruction -**Example:** ``weightslab/examples/PyTorch/ws-generation/main.py`` +**Example:** ``weightslab/examples/PyTorch/wl-generation/main.py`` **Task:** Unsupervised anomaly detection on MVTec capsule images with a multi-task UNet (classification head + reconstruction head + contrastive loss). @@ -99,5 +99,5 @@ filter by pair distance to find the hardest negatives. .. code-block:: bash - weightslab ui launch # 1. deploy the studio + weightslab start # 1. deploy the studio weightslab start example --gen # 2. start the generation / anomaly demo diff --git a/docs/examples/pytorch/segmentation.rst b/docs/examples/pytorch/segmentation.rst index 14fb5c00..68ec2b37 100644 --- a/docs/examples/pytorch/segmentation.rst +++ b/docs/examples/pytorch/segmentation.rst @@ -12,7 +12,7 @@ Segmentation — BDD100k (PyTorch) dense prediction -**Example:** ``weightslab/examples/PyTorch/ws-segmentation/main.py`` +**Example:** ``weightslab/examples/PyTorch/wl-segmentation/main.py`` **Task:** Per-instance semantic segmentation on BDD100k (6 classes) with a small UNet. @@ -112,5 +112,14 @@ these signals over training steps and lets you sort samples by them. .. code-block:: bash - weightslab ui launch # 1. deploy the studio + weightslab start # 1. deploy the studio weightslab start example --seg # 2. start the segmentation demo + + +.. raw:: html + +
+ + Open In Colab + +
diff --git a/docs/examples/ultralytics/detection.rst b/docs/examples/ultralytics/detection.rst index 59577aac..14168e47 100644 --- a/docs/examples/ultralytics/detection.rst +++ b/docs/examples/ultralytics/detection.rst @@ -16,7 +16,7 @@ The full Ultralytics integration documentation is at :doc:`/ultralytics`. This page summarises what ``WLAwareTrainer`` handles automatically and where the example lives. -**Example:** ``weightslab/examples/Ultralytics/ws-detection/main.py`` +**Example:** ``weightslab/examples/Ultralytics/wl-detection/main.py`` What the example does --------------------- diff --git a/docs/examples/usecases/lidar_detection.rst b/docs/examples/usecases/lidar_detection.rst index b1b667e6..b86b13a5 100644 --- a/docs/examples/usecases/lidar_detection.rst +++ b/docs/examples/usecases/lidar_detection.rst @@ -14,8 +14,8 @@ LiDAR Detection — 2D and 3D (PyTorch) **Examples:** -- ``weightslab/examples/Usecases/ws-2d-lidar-detection/main.py`` -- ``weightslab/examples/Usecases/ws-3d-lidar-detection/main.py`` +- ``weightslab/examples/Usecases/wl-2d-lidar-detection/main.py`` +- ``weightslab/examples/Usecases/wl-3d-lidar-detection/main.py`` **Task:** Object detection on LiDAR point clouds — 2D pillar-grid (BEV) and full 3D bounding boxes (KITTI-format). @@ -120,7 +120,7 @@ and override ``load_points`` and optionally ``render_thumbnail_2d``: .. code-block:: bash - weightslab ui launch # 1. deploy the studio + weightslab start # 1. deploy the studio weightslab start example --2d_det # 2a. 2D pillar-grid detection # or weightslab start example --3d_det # 2b. 3D bounding-box detection diff --git a/docs/examples/usecases/loss_shape_classification.rst b/docs/examples/usecases/loss_shape_classification.rst index 6864322a..d5240632 100644 --- a/docs/examples/usecases/loss_shape_classification.rst +++ b/docs/examples/usecases/loss_shape_classification.rst @@ -12,7 +12,7 @@ Loss-Shape Classification per Sample trajectory -**Example:** ``weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/`` +**Example:** ``weightslab/examples/Usecases/wl-loss_shapes_classification_per_sample/`` This use case builds on :doc:`../pytorch/detection` (same Penn-Fudan dataset, same model, same ``guard_training_context`` pattern) and adds one new feature: @@ -203,8 +203,17 @@ Workflow in the studio .. code-block:: bash - weightslab ui launch # 1. deploy the studio + weightslab start # 1. deploy the studio weightslab start example --det # 2. start the detection demo To run the full loss-shape example (with the ``@wl.signal(subscribe_to=...)`` classifier active), use the direct path above. + + +.. raw:: html + +
+ + Open In Colab + +
diff --git a/docs/four_way_approach.rst b/docs/four_way_approach.rst index 9981dbbe..effa9d2d 100644 --- a/docs/four_way_approach.rst +++ b/docs/four_way_approach.rst @@ -52,7 +52,7 @@ Typical integration flow to custom Python script .. code-block:: bash - weightslab ui launch - python train.py + weightslab start + python train.py - Resume training from the UI and use tags/discards/signals to iteratively improve data and model behavior. diff --git a/docs/index.rst b/docs/index.rst index 84a14199..af68b009 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -72,7 +72,7 @@ Weightslab is a Python SDK to inspect, monitor, and edit training behavior for c :link: weights_studio :link-type: doc - Deploy and operate the UI: architecture, Docker, ports, and actions. + Deploy and operate the UI: architecture, ports, TLS, and actions. .. grid-item-card:: Configuration :link: configuration @@ -112,7 +112,6 @@ Weightslab is a Python SDK to inspect, monitor, and edit training behavior for c :hidden: usage/good_practice - usage/docker .. toctree:: diff --git a/docs/logger.rst b/docs/logger.rst index 33ee71e7..e4180736 100644 --- a/docs/logger.rst +++ b/docs/logger.rst @@ -50,6 +50,108 @@ for per-annotation values (detection / segmentation). The signal name comes from ``log=False`` to persist per-sample values without a dashboard curve. See :doc:`user_functions` for the full wrapper reference. +Loss-shape classification +-------------------------- + +A single scalar hides how a sample got there. The *shape* of a per-sample +loss trajectory over training — steadily dropping, stuck, forgotten, noisy — +tells you whether the model is learning that sample, struggling with it, or +whether it's a candidate mislabel. Weightslab classifies every sample's +trajectory into one of six shapes: + +============== ==================================================================== +Label Meaning +============== ==================================================================== +monotonic Loss steadily decreasing — the model is learning the sample. +plateaued Decreased then leveled off still-high — stuck / hard sample. +Flat_high Never moved, stayed high — likely a mislabel or unlearnable. +high_variance Noisy oscillation — model uncertain, often an ambiguous label. +U_Shape Learned then forgotten — catastrophic interference from later data. +Spiked Sudden jump at some step — data/augmentation/version change. +============== ==================================================================== + +Automatic — zero setup +~~~~~~~~~~~~~~~~~~~~~~~ + +Any signal wrapped with ``flag="loss"`` (as above) is classified +automatically; there is nothing to call. The logger runs a background thread +(off the training thread; interval via ``WL_LOGGER_FLUSH_INTERVAL_SECONDS``, +default 2s) that discovers every ``flag="loss"`` signal and, once a sample has +enough history to classify (5+ points), tags it with a ``_shape`` +categorical tag — e.g. ``train_loss/CE`` gets a ``train_loss/CE_shape`` tag: + +.. code-block:: python + + train_loss = wl.watch_or_edit( + nn.CrossEntropyLoss(reduction="none"), + flag="loss", signal_name="train_loss/CE", log=True, + ) + # Nothing else needed — train_loss/CE_shape starts appearing on samples + # as they accumulate enough history, refreshed every background tick. + +``flag="metric"`` signals are **not** auto-classified: the default classifier +assumes a decreasing trajectory (a loss), which would misclassify an +increasing metric like accuracy. + +Overriding or opting out +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use :func:`enable_loss_shape_autotag` / :func:`disable_loss_shape_autotag` +only to customize or opt out — not to turn classification on: + +.. code-block:: python + + # Different tag name / classifier for one signal (e.g. it isn't a loss). + wl.enable_loss_shape_autotag("train_loss/CE", tag_name="curve_shape") + + # Opt in a signal that wasn't registered via flag="loss" (e.g. logged + # manually through wl.save_signals under a custom name). + wl.enable_loss_shape_autotag("custom_metric") + + # Opt out one signal, or every signal. + wl.disable_loss_shape_autotag("train_loss/CE") + wl.disable_loss_shape_autotag() + +:func:`auto_loss_shape_signal_names` lists every signal currently tracked. + +One-off / report-time classification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To force a synchronous, guaranteed-fresh classification right before a +specific report, pass ``loss_shape_signal`` to :func:`write_dataframe`, or +call the underlying reusable engine directly — it works for **any** +per-sample signal, not just a loss (pass your own ``classifier`` for an +increasing metric such as accuracy): + +.. code-block:: python + + wl.write_dataframe("report.csv", format="csv", loss_shape_signal="train_loss/CE") + + wl.write_signal_shapes("train_loss/CE", tag_name="train_loss/CE_shape") + wl.write_loss_shapes("train_loss/CE") # convenience wrapper (tag: loss_shape) + +Building a custom classifier +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:func:`trajectory_stats` computes scale-invariant features (net drop, +coefficient of variation, argmin fraction, rebound, max jump, tail flatness) +from a sample's ordered value history. Reuse them in your own rule, or pass a +full ``classifier`` callable (``list[float] -> str | None``) to any function +above: + +.. code-block:: python + + def my_classifier(values): + s = wl.trajectory_stats(values) + return None if s is None else ("fast" if s["drop"] > 0.6 else "slow") + + wl.enable_loss_shape_autotag("train_loss/CE", classifier=my_classifier) + +For a live, per-step reactive variant (heavier — reads history on every fire) +via :func:`enable_loss_shape_signal`, or a fully hand-rolled +``@wl.signal(subscribe_to=...)`` walkthrough, see +:doc:`examples/usecases/loss_shape_classification`. + Custom signals -------------- diff --git a/docs/pytorch_lightning.rst b/docs/pytorch_lightning.rst index 3a0aa64b..5643983d 100644 --- a/docs/pytorch_lightning.rst +++ b/docs/pytorch_lightning.rst @@ -3,7 +3,7 @@ PyTorch Lightning Integration Weightslab is compatible with PyTorch Lightning and already includes a full example: -``weightslab/examples/PyTorch_Lightning/ws-classification/main.py`` +``weightslab/examples/PyTorch_Lightning/wl-classification/main.py`` This page explains how to integrate Weightslab in a Lightning workflow and scale to multiple GPUs. @@ -109,7 +109,7 @@ Optional YAML-driven trainer config The Lightning example already includes a ready template at: -``weightslab/examples/PyTorch_Lightning/ws-classification/config.yaml`` +``weightslab/examples/PyTorch_Lightning/wl-classification/config.yaml`` .. code-block:: yaml diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 08fa3be4..d0ee011d 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -7,9 +7,8 @@ If you prefer to start from examples, see ``usecases`` right after this setup. Prerequisites ------------- -- Python v3.10+ installed. -- A virtual environment tool like ``venv`` or Conda (optionnal). -- Docker v4+ to start the UI. +- Python v3.10+ installed +- A virtual environment tool like ``venv`` or Conda (optional). - Your training project available locally. Install WeightsLab @@ -38,13 +37,13 @@ example like the classification example (--cls). It run a small experiment on a .. code-block:: bash - weightslab example start --cls + weightslab start example --cls -Then, in another terminal, launch the UI and open http://localhost:5173: +Then, in another terminal, launch the UI and open the URL printed by the command: .. code-block:: bash - weightslab ui launch + weightslab start .. Launch WeightsLab services from your training script @@ -139,8 +138,8 @@ Pass ``--certs`` to generate (if missing) and use TLS certificates + a gRPC auth .. code-block:: bash - weightslab ui launch # unsecured HTTP (default; no certs generated) - weightslab ui launch --certs # secured HTTPS + gRPC auth (generates certs if missing) + weightslab start # unsecured HTTP (default) + weightslab start --certs # secured HTTPS + gRPC auth (run `weightslab se` first) .. important:: @@ -148,13 +147,9 @@ Pass ``--certs`` to generate (if missing) and use TLS certificates + a gRPC auth terminal use the **same** certificates — it is the single source of truth for TLS/auth. Please note that this step has to be done before starting the experiment. Run ``weightslab``, ``weightslab help``, or ``weightslab -h`` to see the banner and the full -command reference (``se``, ``ui launch``, ``start example ...``). +command reference (``se``, ``start``, ``start example ...``). -To stop the UI later: - -.. code-block:: bash - - docker stop weights_studio_envoy weights_studio_frontend +To stop the UI, press ``Ctrl+C`` in the terminal running ``weightslab start``. Prefer a terminal over a browser? ``weightslab cli`` opens an interactive console connected to the running experiment (pause/resume, status, evaluate, diff --git a/docs/segmentation_usecase.rst b/docs/segmentation_usecase.rst index 38738a54..5fe8b83b 100644 --- a/docs/segmentation_usecase.rst +++ b/docs/segmentation_usecase.rst @@ -3,7 +3,7 @@ Segmentation Use Case — Per-instance & Per-sample Signals (PyTorch) This page walks through the segmentation integration from: -``weightslab/examples/PyTorch/ws-segmentation/main.py`` +``weightslab/examples/PyTorch/wl-segmentation/main.py`` It builds on the classification :doc:`usecases` page and focuses on what is **specific to segmentation**: a *list of per-instance masks* per sample, a custom diff --git a/docs/ultralytics.rst b/docs/ultralytics.rst index 18af0fd3..5b7c214a 100644 --- a/docs/ultralytics.rst +++ b/docs/ultralytics.rst @@ -8,7 +8,7 @@ without touching the model or YOLO's training loop. The full example lives at: -``weightslab/examples/Ultralytics/ws-detection/`` +``weightslab/examples/Ultralytics/wl-detection/`` How it works ------------ @@ -201,15 +201,15 @@ Running the bundled example .. code-block:: bash - weightslab ui launch + weightslab start 4. Run the example: .. code-block:: bash - python weightslab/examples/Ultralytics/ws-detection/main.py + python weightslab/examples/Ultralytics/wl-detection/main.py -5. Open ``http://localhost:5173`` to monitor training, inspect per-sample +5. Open the URL printed by ``weightslab start`` to monitor training, inspect per-sample signals, tag difficult images, and discard outliers. Platform notes diff --git a/docs/usage/docker.rst b/docs/usage/docker.rst deleted file mode 100644 index 26beb6d8..00000000 --- a/docs/usage/docker.rst +++ /dev/null @@ -1,350 +0,0 @@ -.. _docker-usage: - -Docker Usage -=============== - -**Examples:** ``weightslab/examples/Docker_training/`` - -WeightsLab supports running the training script — and the entire UI stack — -inside Docker. Three integration patterns are available; they differ in **how -the trainer container gets a Docker daemon** to launch the Envoy proxy and the -Weights Studio frontend alongside it. - -.. list-table:: - :header-rows: 1 - :widths: 28 24 24 24 - - * - - - **A · Docker-in-Docker** - - **B · Siblings (DooD)** - - **C · Self-contained siblings** - * - Docker daemon - - Own nested daemon - - Host daemon (socket mount) - - Host daemon (socket mount) - * - Envoy/frontend run - - Nested inside trainer - - Siblings on the host - - Siblings on the host - * - Starts UI via - - ``weightslab ui launch`` - - ``weightslab ui launch`` - - Custom ``ui-compose.yml`` - * - ``--privileged`` - - **Required** - - No - - No - * - gRPC ``:50051`` published - - Not published - - Yes - - Yes - * - Host bind mounts - - None (co-located fs) - - Path alignment required - - **None** - * - Host setup - - None - - ``setup-host.sh`` - - **None** - * - HTTPS (optional) - - ``WEIGHTSLAB_TLS=1`` - - ``WEIGHTSLAB_TLS=1`` - - ``WEIGHTSLAB_TLS=1`` - * - Windows / Docker Desktop - - Yes - - Awkward (use WSL2) - - **Yes** - -.. note:: - - - **A (DinD)** — fully self-contained; needs ``--privileged``. Best when - you want isolation or are on Windows. - - **B (DooD)** — uses the stock ``weightslab ui launch``; cleanest on a - Linux host with path alignment already done. - - **C (self-contained siblings)** — no host prep, no bind mounts, - works natively on Windows. - -.. _docker-dind: - -Option A: Docker-in-Docker (DinD) ----------------------------------- - -**Source:** ``weightslab/examples/Docker_training/docker_in_docker/`` - -The trainer container starts its **own inner Docker daemon** and uses -``weightslab ui launch`` inside it. Because the inner daemon shares the -trainer container's filesystem, all paths resolve without any host-side setup. - -Wiring diagram -~~~~~~~~~~~~~~ - -.. code-block:: text - - host browser - → localhost:5173 ── (re-published) ──► trainer ──► inner frontend :5173 - → localhost:8080 ── (re-published) ──► trainer ──► inner Envoy :8080 - │ grpc-backend:host-gateway - ▼ - in-process gRPC backend :50051 - (same container as trainer) - -Configuration requirements -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: 5 40 30 25 - - * - # - - Requirement - - Location - - Why - * - 1 - - ``privileged: true`` on the trainer service - - ``docker-compose.yml`` - - inner ``dockerd`` cannot run otherwise - * - 2 - - Start the inner ``dockerd`` + wait for socket - - ``entrypoint.sh`` - - nested daemon that hosts Envoy + frontend - * - 3 - - Persist ``/var/lib/docker`` with a named volume - - ``docker-compose.yml`` - - caches image pulls across runs - * - 4 - - Publish ``5173`` + ``8080`` to the host - - ``docker-compose.yml`` - - browser reaches the *nested* containers - * - 5 - - ``GRPC_BACKEND_PORT=50051`` - - ``docker-compose.yml`` - - Envoy's ``grpc-backend:host-gateway`` dials this port - * - 6 - - ``WEIGHTSLAB_SKIP_DOCKER_OPS=1`` before ``ui launch`` - - ``entrypoint.sh`` - - skips the in-container rebuild; pulls the image only - * - 7 - - Order: ``ui launch`` → ``start example`` - - ``entrypoint.sh`` - - UI stack must be up before the backend starts - -GPU access -~~~~~~~~~~ - -Add the ``deploy`` block to the trainer service in ``docker-compose.yml``: - -.. code-block:: yaml - - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] - -The host also needs the **NVIDIA Container Toolkit** installed (``sudo -nvidia-ctk runtime configure --runtime=docker``). Comment out the ``deploy`` -block on hosts without an NVIDIA GPU/toolkit — ``docker compose up`` will -otherwise fail with *"could not select device driver nvidia"*. - -Running -~~~~~~~ - -.. code-block:: bash - - # from weightslab/examples/Docker_training/docker_in_docker/ - docker compose up --build - -Open http://localhost:5173. Stop with ``Ctrl+C`` or ``docker compose down``. - -.. dropdown:: Enable HTTPS / mTLS (optional) - :color: secondary - - Only needed for remote or production access. For local development, plain - HTTP at http://localhost:5173 works without any certificates. - - DinD is the simplest TLS option because certs, Envoy, and the backend are - co-located: - - .. code-block:: bash - - WEIGHTSLAB_TLS=1 docker compose up --build - - Then trust the generated CA on the host browser (once): - - .. code-block:: powershell - - # Windows — pull the CA out of the running container and trust it - docker cp weightslab_trainer_dind:/root/.weightslab-certs/ca.crt . - Import-Certificate -FilePath .\ca.crt -CertStoreLocation Cert:\CurrentUser\Root - - Open https://localhost:5173. - - What ``WEIGHTSLAB_TLS=1`` does in ``entrypoint.sh``: - - 1. Runs ``weightslab ui launch --certs``, generating certs into - ``WEIGHTSLAB_CERTS_DIR`` and configuring Envoy + the frontend for HTTPS. - 2. Exports ``GRPC_TLS_ENABLED=1`` + ``GRPC_TLS_CERT_DIR`` so the gRPC - backend also speaks TLS (required to prevent Envoy upstream 503s). - -.. _docker-siblings-c: - -Option C: Self-contained siblings (recommended for Windows) ------------------------------------------------------------- - -**Source:** ``weightslab/examples/Docker_training/siblings_self_contained/`` - -The trainer mounts the **host Docker socket** and starts Envoy + the frontend -as sibling containers — but unlike Option B, it **never bind-mounts a host -path**. The Envoy config (and, for TLS, the certs) are delivered via **named -volumes over the socket**, so this option works on **Windows / Docker Desktop** -with zero host preparation. - -Wiring diagram -~~~~~~~~~~~~~~ - -.. code-block:: text - - host browser - → localhost:5173 ──► frontend container (sibling, HTTP) - → localhost:8080 ──► Envoy container (sibling, config from named volume) - │ grpc-backend:host-gateway → host:50051 - ▼ - host:50051 ──► trainer's gRPC backend :50051 - -Configuration requirements -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: 5 40 30 25 - - * - # - - Requirement - - Location - - Why - * - 1 - - Mount ``/var/run/docker.sock`` - - ``docker-compose.yml`` - - drive the host daemon (start sibling containers) - * - 2 - - Publish ``50051:50051`` from the trainer - - ``docker-compose.yml`` - - sibling Envoy dials it via ``host-gateway`` - * - 3 - - Stage Envoy config into ``wl_envoy_cfg`` named volume - - ``entrypoint.sh`` - - replaces the host bind mount — no host path needed - * - 4 - - ``ui-compose.yml`` with no host bind mounts - - ``ui-compose.yml`` - - nothing for the host daemon to resolve on disk - * - 5 - - ``extra_hosts: grpc-backend:host-gateway`` on the Envoy service - - ``ui-compose.yml`` - - routes Envoy → host:50051 → trainer - * - 6 - - ``GRPC_BACKEND_PORT=50051`` - - ``docker-compose.yml`` - - port the backend binds + Envoy dials - -How config delivery works without bind mounts -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Option C does not call ``weightslab ui launch``. Instead, ``entrypoint.sh``: - -1. Renders Envoy's plaintext config from the installed ``weightslab`` package. -2. Pipes it into the ``wl_envoy_cfg`` named volume via the socket using a - temporary ``busybox`` container — no host path involved. -3. Brings up Envoy + frontend using ``ui-compose.yml``, which reads the config - from the named volume rather than a bind-mounted host directory. - -GPU access -~~~~~~~~~~ - -Same as Option A — add the ``deploy`` block to the trainer service and install -the NVIDIA Container Toolkit on the host. - -Running -~~~~~~~ - -.. code-block:: bash - - # from weightslab/examples/Docker_training/siblings_self_contained/ - docker compose up --build - -Open http://localhost:5173. - -**Stopping:** - -.. code-block:: bash - - docker compose down # stop trainer - docker compose -p weightslab_ui -f ui-compose.yml down # stop UI siblings - docker volume rm wl_envoy_cfg # optional: drop staged config - -.. dropdown:: Enable HTTPS / mTLS (optional) - :color: secondary - - Only needed for remote or production access. For local development, plain - HTTP at http://localhost:5173 works without any certificates. - - TLS works here without host bind mounts — certs are piped into named volumes - the same way as ``envoy.yaml``: - - .. code-block:: bash - - WEIGHTSLAB_TLS=1 docker compose up --build - - Then trust the CA on the host browser (once): - - .. code-block:: powershell - - docker cp weightslab_trainer_selfcontained:/root/.weightslab-certs/ca.crt . - Import-Certificate -FilePath .\ca.crt -CertStoreLocation Cert:\CurrentUser\Root - - Open https://localhost:5173. - - What ``WEIGHTSLAB_TLS=1`` delivers (all via named volumes, no host paths): - - .. list-table:: - :header-rows: 1 - :widths: 35 30 35 - - * - Layer - - What's needed - - How it arrives - * - Browser ↔ Envoy - - ``envoy-server.crt/key`` - - ``wl_envoy_cfg`` volume - * - Envoy ↔ backend (mTLS) - - ``envoy-client.crt/key`` + ``ca.crt`` - - ``wl_envoy_cfg`` volume - * - Browser ↔ frontend - - ``envoy-server.crt/key`` - - ``wl_nginx_certs`` volume - * - Backend gRPC server - - ``backend-server.crt/key`` + ``ca.crt`` - - ``GRPC_TLS_ENABLED=1`` + ``GRPC_TLS_CERT_DIR`` - - .. note:: - - Generated keys are ``0600`` (owned by root). ``entrypoint.sh`` runs - ``chmod a+rX`` on cert files in the volumes so Envoy (non-root) can - read them. - -Common notes ------------- - -- The example starts **paused** (``is_training: false``); start and steer - training from the UI at http://localhost:5173. -- First run is slow: it pulls ``envoyproxy/envoy`` + ``graybx/weightslab`` - and downloads MNIST for the classification example. -- For either option, to build against the dev branch instead of PyPI: - - .. code-block:: bash - - docker compose build \ - --build-arg WEIGHTSLAB_SPEC="git+https://github.com/GrayboxTech/weightslab.git@dev" - docker compose up diff --git a/docs/usage/parameters.rst b/docs/usage/parameters.rst index c23ae9c9..c7fb22f0 100644 --- a/docs/usage/parameters.rst +++ b/docs/usage/parameters.rst @@ -460,36 +460,6 @@ Audit logging interaction with the studio. Accepted: ``json``, ``csv``, ``none`` (disables audit logging). -Docker integration -~~~~~~~~~~~~~~~~~~~ - -These variables are set inside Docker training containers; see -:ref:`docker-usage` for full context. - -.. list-table:: - :header-rows: 1 - :widths: 35 15 50 - - * - Variable - - Default - - Description - * - ``GRPC_BACKEND_PORT`` - - ``50051`` - - Port the gRPC backend binds to. Must match the port Envoy - is configured to dial as ``grpc-backend``. - * - ``WEIGHTSLAB_TLS`` - - ``0`` - - Set to ``1`` inside a Docker Compose stack to enable the full - TLS + cert-generation flow (DinD and self-contained siblings). - * - ``WEIGHTSLAB_SKIP_DOCKER_OPS`` - - ``0`` - - Set to ``1`` inside a DinD container before ``weightslab ui - launch`` to skip the image rebuild and pull only. - * - ``WS_SERVER_PROTOCOL`` - - ``http`` - - Protocol served by the frontend's nginx. Set to ``https`` - when providing TLS certificates to the Weights Studio container. - LLM / agent integration (optional) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -511,6 +481,16 @@ LLM / agent integration (optional) * - ``OPENROUTER_REQUEST_TIMEOUT`` - *(unset)* - Per-request timeout in seconds for OpenRouter calls. + * - ``OPENROUTER_MAX_TOKENS`` + - ``2048`` + - Maximum completion length requested from OpenRouter. OpenRouter + pre-authorizes ``max_tokens × completion_price`` against the key's + remaining budget *before* generating, so leaving this uncapped makes + the model request its full output window and can fail with a ``402`` + ("requires more credits, or fewer max_tokens") on a credit- or + weekly-limited key — even though the model is otherwise usable. The + default is ample for intent planning; raise it only if you see + truncated responses. Telemetry ~~~~~~~~~~ diff --git a/docs/usecases.rst b/docs/usecases.rst index 307f167c..8de11fbb 100644 --- a/docs/usecases.rst +++ b/docs/usecases.rst @@ -3,7 +3,7 @@ Use Case Example (PyTorch) This page walks through the real MNIST classification integration from: -``weightslab/examples/PyTorch/ws-classification/main.py`` +``weightslab/examples/PyTorch/wl-classification/main.py`` Goal ---- diff --git a/docs/user_commands.rst b/docs/user_commands.rst index 0b3d3fcb..c8845d5e 100644 --- a/docs/user_commands.rst +++ b/docs/user_commands.rst @@ -1,56 +1,39 @@ User Commands Reference -======================== +======================= -This page documents the ``weightslab`` command-line tool — every subcommand, -flag, and default — plus the interactive CLI console it (and -``wl.serve(serving_cli=True)``) opens onto a running experiment. +This page documents the weightslab command-line interface and its subcommands. -It's the CLI counterpart to :doc:`user_functions` (the Python API); see that -page for everything you call from inside a training script. +weightslab command +------------------ -Two different things are called "CLI" in WeightsLab — keep them distinct: - -- The **installed command** — ``weightslab ...`` — run from any shell to - manage Docker/the UI, run bundled examples, or connect to a running - experiment. -- The **interactive console** — the REPL you land in after ``weightslab - cli`` (or a standalone ``python -m weightslab.backend.cli client``) - connects — where you type commands like ``pause``, ``status``, or - ``evaluate``. - -``weightslab`` command ------------------------ - -Installed as a console script (``weightslab = weightslab.ui_docker_bridge:main``), -so it's available anywhere the package is installed — no ``python -m`` needed. +Installed as a console script via pyproject.toml: .. code-block:: text - weightslab {se,ui,start,cli,help} ... + weightslab {se,start,cli,tunnel,help} ... -Running ``weightslab``, ``weightslab -h`` / ``--help``, or ``weightslab help`` -with no further arguments prints the banner and this same command summary. +Run weightslab, weightslab -h, or weightslab help to print the full built-in help. .. list-table:: :header-rows: 1 * - Command - Purpose - * - ``weightslab se`` - - One-time secure setup: generate TLS certs + a gRPC auth token. - * - ``weightslab ui launch`` - - Clean stale Docker state, then build & start the Weights Studio UI stack. - * - ``weightslab start example`` (alias: ``weightslab example start``) - - Run a bundled PyTorch example in the foreground. - * - ``weightslab cli`` - - Open an interactive console connected to a running experiment. - * - ``weightslab help`` - - Show the help/banner (same as no command, or ``-h``). + * - weightslab se + - Generate TLS certificates and gRPC auth token in WEIGHTSLAB_CERTS_DIR. + * - weightslab start + - Start the native Weights Studio server (bundled SPA + gRPC-Web proxy). + * - weightslab start example + - Run a bundled training example. + * - weightslab cli + - Connect to a running experiment interactive console. + * - weightslab tunnel + - Forward a remote gRPC backend to a local TCP port. + * - weightslab help + - Show the help/banner (same as no command, or -h). weightslab se -~~~~~~~~~~~~~~ - -**Syntax** +~~~~~~~~~~~~~ .. code-block:: bash @@ -58,81 +41,43 @@ weightslab se Generates TLS certificates and a gRPC auth token into a certs directory, then tells you to export ``WEIGHTSLAB_CERTS_DIR`` — the **single source of -truth** the training backend, ``weightslab ui launch --certs``, and any new +truth** the training backend, ``weightslab start --certs``, and any new shell all read to decide whether TLS/auth is on (derived purely from whether cert files exist in that directory). -**Arguments** - -- ``certs_dir`` *(positional, optional)* — custom directory for the certs + - token. Default: ``$WEIGHTSLAB_CERTS_DIR`` if already set in the - environment, else ``~/.weightslab-certs``. -- ``--force-certs`` — regenerate certificates even if valid ones already - exist in the target directory. Default: off (existing certs are reused). - -**Examples** +weightslab start +~~~~~~~~~~~~~~~~ .. code-block:: bash - weightslab se # one-time secure setup - weightslab se --force-certs # regenerate the certs - weightslab se /custom/certs/path # use a custom directory + weightslab start [--port PORT] [--config FILE] [--host HOST] + [--backend-host HOST] [--backend-port PORT] + [--no-browser] [--certs] -After it finishes, export the printed ``WEIGHTSLAB_CERTS_DIR`` value -permanently (the command prints the exact ``export`` / ``setx`` line for -your platform) so the training backend and Weights Studio agree on the same -certificates. +Runs the UI natively from Python. -weightslab ui launch -~~~~~~~~~~~~~~~~~~~~~~ +Port resolution order: -**Syntax** +1. --port +2. ui_port from --config / WEIGHTSLAB_EXPERIMENT_CONFIG config file +3. WL_LAST_UI_PORT +4. WEIGHTSLAB_UI_PORT (compatibility) +5. 50051 -.. code-block:: bash - - weightslab ui launch [certs_dir] [--certs] [-i/--image REPO] [-v/--version TAG] - -Purges stale ``weightslab``/``weights_studio`` Docker resources scoped to the -bundled stack, then builds and starts the Weights Studio UI via Docker -Compose. - -**Arguments** +If the chosen port is already in use, weightslab start falls back to a random +available port and logs it. -- ``certs_dir`` *(positional, optional)* — same meaning as for ``weightslab se``. -- ``--certs`` — generate certs (if missing) and run **secured** (HTTPS + - gRPC auth). Default: off — the UI launches **unsecured** (plain HTTP, no - gRPC auth, no certs generated). Existing certs already present in - ``WEIGHTSLAB_CERTS_DIR`` are always honored either way and are never - deleted by this command. -- ``-i``, ``--image`` *(str, optional)* — frontend image repo to run/pull. - Default: ``graybx/weightslab``. -- ``-v``, ``--version`` *(str, optional)* — frontend image tag/version to - pull. Default: ``latest``. An explicit ``--version`` overrides any tag - embedded in ``--image``. - -**Examples** +Examples: .. code-block:: bash - weightslab ui launch # unsecured HTTP (default) - weightslab ui launch --certs # secured HTTPS + gRPC auth - weightslab ui launch -i guillaumep2705/weightslab # pull a custom repo (latest tag) - weightslab ui launch -i guillaumep2705/weightslab -v v1.2.3 # pin a specific version - -Once running, the UI is served at ``http://localhost:5173`` (or -``https://...`` when ``--certs`` is used); the exact URL is also printed at -the end of the command. - -.. important:: - - When using ``--certs``, set ``WEIGHTSLAB_CERTS_DIR`` manually (before - starting your training script) so the training backend and this UI use - the **same** certificates. + weightslab start + weightslab start --port 9000 + weightslab start --backend-port 50052 + weightslab start --certs weightslab start example -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Syntax** +~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: bash @@ -178,39 +123,89 @@ the documented form. weightslab start example --3d_det # 3D LiDAR detection weightslab example start --det # tolerant alias, same as `start example --det` -Then, in another terminal: ``weightslab ui launch`` and open +Then, in another terminal: ``weightslab launch`` and open ``http://localhost:5173``. See :doc:`examples/index` for what each example demonstrates. weightslab cli -~~~~~~~~~~~~~~~ - -**Syntax** +~~~~~~~~~~~~~~ .. code-block:: bash weightslab cli [--port PORT] [--host HOST] -Opens the :ref:`interactive console ` connected to a -**currently-running experiment** — the experiment must have called -``wl.serve(serving_cli=True)`` (the default when calling ``wl.serve()`` with -no arguments). +Connects to a running experiment CLI server. + +weightslab tunnel +~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + weightslab tunnel [ENDPOINT] [--listen-port N] [--listen-host H] [--remote-port N] + +Forwards a remote gRPC endpoint to a local TCP port. + +- Use raw TCP tunnels (for example bore or ngrok tcp). +- Default local listen port is 50051. + +- The remote tunnel must be **raw TCP**, *not* an HTTP/gRPC-Web tunnel. A + zero-signup option is `bore `_ with its free + public relay: ``bore local 50051 --to bore.pub`` (prints ``bore.pub:``). + ``ngrok tcp 50051`` also works but now requires a credit card on the free tier. +- The backend must run **plaintext** — the default ``weightslab launch`` + (no ``--certs``) — so no TLS terminates mid-path. **Arguments** -- ``--port`` *(int, optional)* — connect to a specific CLI server port. - Default: auto-discover the running experiment (the backend advertises its - actual host/port on startup; ``weightslab cli`` reads that advertisement). -- ``--host`` *(str, optional)* — connect to a specific host. Default: - ``localhost``. +- ``ENDPOINT`` *(positional, optional)* — the remote backend as ``host:port`` + (e.g. ``0.tcp.ngrok.io:12345``); a ``tcp://`` prefix is accepted and + stripped. Default: the ``WEIGHTSLAB_TUNNEL_ENDPOINT`` environment variable, so + a bare ``weightslab tunnel`` works once that is exported. +- ``--listen-port``, ``-p`` *(int)* — local port to expose. Default: **50051** + (the port the bundled Envoy upstream dials — leave it unless you changed + ``GRPC_BACKEND_PORT``). +- ``--listen-host`` *(str)* — interface to bind. Default: **auto** — + ``127.0.0.1`` on Windows/macOS (Docker Desktop reaches host loopback via + ``host.docker.internal``), ``0.0.0.0`` on Linux (compose ``host-gateway`` + resolves to the bridge IP, which cannot reach a loopback-only listener). +- ``--remote-port`` *(int)* — the remote port, when ``ENDPOINT`` has only a + host and no ``:port``. **Examples** .. code-block:: bash - weightslab cli # auto-discover the running experiment - weightslab cli --port 60000 # connect to a specific port - weightslab cli --host 10.0.0.5 --port 60000 + weightslab tunnel bore.pub:12345 # bridge remote backend -> localhost:50051 + weightslab tunnel tcp://bore.pub:12345 # tcp:// prefix is fine + weightslab tunnel # uses $WEIGHTSLAB_TUNNEL_ENDPOINT + weightslab tunnel host.example.com --remote-port 50051 + weightslab tunnel host:50051 -p 50055 # expose locally on a different port + +**Typical workflow** (Colab backend, local UI): + +.. code-block:: bash + + # 1) In Colab: expose the training backend over raw TCP (prints bore.pub:) + # !bore local 50051 --to bore.pub + + # 2) On your machine, in two terminals: + weightslab launch # plaintext HTTP (default) + weightslab tunnel bore.pub:12345 # in another window, the host:port bore printed + + # 3) Open http://localhost:5173 — Studio streams live from Colab. + +.. note:: + + Step 1 can be done for you: call ``wl.serve(serving_grpc=True, + serving_bore=True)`` in the training script. It downloads ``bore``, opens the + relay, and prints the exact ``weightslab tunnel bore.pub:`` line to run + on your machine — see ``serve`` in :doc:`user_functions`. + +The command probes the remote on startup (warning, not fatal, if it isn't up +yet), re-resolves the endpoint per connection (so a changing tunnel IP is picked +up), and runs until ``Ctrl+C``. See the classification Colab notebook +(``examples/Notebooks/PyTorch/wl-classification.ipynb``) for the end-to-end +setup. .. _cli-console: diff --git a/docs/user_functions.rst b/docs/user_functions.rst index e3a614ad..4668e942 100644 --- a/docs/user_functions.rst +++ b/docs/user_functions.rst @@ -323,6 +323,7 @@ signal name: str, subscribe_to: str, compute_every_n_steps: int = 1, + min_step: int = 0, include_history: bool = False, include_history_metadata: bool = False ) @@ -345,6 +346,11 @@ sorting and root-cause analysis in the studio). ``ctx.subscribed_value``. If omitted, the signal is **static**. - ``compute_every_n_steps``: throttle for dynamic signals (e.g. ``10`` = compute on every 10th step the subscribed metric is produced). +- ``min_step``: minimum training step before a dynamic signal starts firing. + While ``current_step < min_step`` the signal is skipped. Defaults to ``0`` + (fire from the start). Use it when a signal needs enough history to be + meaningful — e.g. a loss-shape classifier that should only run once each + sample has a trajectory (``min_step=505``). **Static vs dynamic** @@ -511,7 +517,7 @@ Spiked Sudden jump at some step — data/augmentation/version change. the verdict also live as a per-sample ``signals//loss_shape_classifier`` column; the human-readable label lives on the ``loss_shape`` categorical tag. - See the detection use case (``examples/PyTorch/ws-detection/src/main.py``) for + See the detection use case (``examples/PyTorch/wl-detection/src/main.py``) for this signal wired into a real training loop. compute_signals @@ -1080,6 +1086,7 @@ write_history graph_name=None, experiment_hash=None, sample_id=None, + orient="columns", instance_id=None, ) @@ -1127,8 +1134,28 @@ Dump signal history to a file for offline analysis or debugging. - ``instance_id`` *(int or list of int, optional)* — restrict per-instance rows to one or more annotation IDs. Has no effect on global or per-sample history. +- ``orient`` *(str, optional)* — JSON layout for each section, forwarded to + ``pandas.DataFrame.to_json``. Default ``"columns"`` (see below — compact, + writes each column name once per section instead of once per row). Pass + ``"records"`` for the row-list-of-dicts shape shown further down. Ignored + for ``format="csv"``. -**JSON output shape** +**JSON output shape (default, ``orient="columns"``)** + +.. code-block:: json + + { + "global": {"graph_name": {"0": "loss"}, "experiment_hash": {"0": "h1"}, "step": {"0": 1}, "metric_value": {"0": 0.42}}, + "sample": {"graph_name": {"0": "loss"}, "experiment_hash": {"0": "h1"}, "sample_id": {"0": "img0"}, "step": {"0": 1}, "metric_value": {"0": 0.38}}, + "instance": {"graph_name": {"0": "iou"}, "experiment_hash": {"0": "h1"}, "sample_id": {"0": "img0"}, "annotation_id": {"0": 1}, "step": {"0": 1}, "metric_value": {"0": 0.81}} + } + +Only the sections selected by *type_of_history* are present in the output. +Each section maps column name -> {row index -> value}; round-trips with +``pandas.read_json(path, orient="columns")`` (or per-section via +``pd.DataFrame(data["global"])``). + +**JSON output shape (``orient="records"``)** .. code-block:: json @@ -1138,7 +1165,9 @@ Dump signal history to a file for offline analysis or debugging. "instance": [{"graph_name": "iou", "experiment_hash": "h1", "sample_id": "img0", "annotation_id": 1, "step": 1, "metric_value": 0.81}] } -Only the sections selected by *type_of_history* are present in the output. +The row-list-of-dicts shape used before ``orient`` was wired up — repeats +every column name once per row, so it's larger on disk for many-row +sections. Pass ``orient="records"`` explicitly to keep using it. **CSV output shape** diff --git a/docs/weights_studio.rst b/docs/weights_studio.rst index 930530cb..d005acec 100644 --- a/docs/weights_studio.rst +++ b/docs/weights_studio.rst @@ -1,10 +1,10 @@ Weights Studio Guide ==================== -Weights Studio is the visual frontend for Weightslab experiments. -It connects to your running Weightslab backend over gRPC-Web via Envoy, -and gives you interactive control over samples, tags, discard/restore actions, -training/audit mode, and training signal plots. +Weights Studio is the visual frontend for WeightsLab experiments. +It ships **inside the Python package** — no Docker, no Envoy. +Running ``weightslab start`` serves the bundled SPA and proxies gRPC-Web to +your training backend, all from one Python process. Architecture ------------ @@ -15,180 +15,184 @@ Architecture Runtime path: -1. Browser UI (Vite app) -2. Envoy proxy (gRPC-Web bridge) -3. Weightslab Python gRPC service +1. Browser (served from ``weightslab start``) +2. ``weightslab start`` — pure-Python HTTP server that: -Project location ----------------- - -Weights Studio source lives in: - -- ``../weights_studio`` - -Key files: - -- Docker compose: ``../weights_studio/docker/docker-compose.yml`` -- Envoy config: ``../weights_studio/envoy/envoy.yaml`` -- Frontend entrypoint: ``../weights_studio/src/main.ts`` -- UI layout: ``../weights_studio/index.html`` - -Quick start (Docker) --------------------- - -1. Start your Weightslab backend (gRPC on host, default port ``50051``). -2. Load environment variables from ``../weights_studio/docker/.env``. -3. Generate local TLS certificates (dev only): + - Serves the pre-built Weights Studio SPA (vendored in ``weightslab/ui/static/``) + - Translates gRPC-Web (browser) to raw gRPC (backend) via an embedded proxy - .. code-block:: powershell +3. WeightsLab Python gRPC service (started by ``wl.serve()``) - # from ../weights_studio/docker - .\generate-dev-certs.ps1 +Quick start +----------- - .. code-block:: bash +1. Install WeightsLab:: - # from ../weights_studio/docker - ./generate-dev-certs.sh + pip install weightslab -4. Start studio stack from ``../weights_studio/docker``: +2. In your training script, start the backend:: - - Envoy - - Frontend (Vite) + import weightslab as wl + wl.serve(serving_grpc=True) + # ... training loop ... + wl.keep_serving() -5. Open Weights Studio in your browser. +3. In another terminal, start the UI:: -Docker services and ports -------------------------- - -From ``../weights_studio/docker/docker-compose.yml`` and ``../weights_studio/envoy/envoy.yaml``: - -- Frontend: ``VITE_PORT`` (default ``5173``) -- Envoy gRPC-Web endpoint over TLS: ``ENVOY_PORT`` (default ``8080``) -- Envoy admin: ``ENVOY_ADMIN_PORT`` (default ``9901``), bound to loopback and not published by default -- Backend target from Envoy: ``host.docker.internal:50051`` - -Default values in ``../weights_studio/docker/.env``: - -- ``VITE_PORT=5173`` -- ``VITE_HISTOGRAM_MAX_BINS=512`` -- ``BB_THUMB_RENDER=10`` (max bounding boxes drawn per thumbnail image, per overlay) -- ``BB_MODAL_RENDER=100`` (max bounding boxes drawn per modal image, per overlay) -- ``WS_SERVER_HOST=localhost`` -- ``WS_SERVER_PORT=8080`` -- ``WS_SERVER_PROTOCOL=https`` -- ``ENVOY_PORT=8080`` -- ``ENVOY_ADMIN_PORT=9901`` -- ``GRPC_BACKEND_PORT=50051`` - -How the frontend endpoint is built ----------------------------------- + weightslab start -In ``src/main.ts``, the UI builds the server URL from: +4. Open the URL printed by ``weightslab start`` in your browser. -- ``WS_SERVER_PROTOCOL`` -- ``WS_SERVER_HOST`` -- ``WS_SERVER_PORT`` +The UI auto-discovers the backend on ``localhost:50051`` (default). +Pass ``--backend-port`` to override:: -This URL is used as gRPC-Web base URL for ``ExperimentServiceClient``. + weightslab start --backend-port 50052 -Environment/configuration checklist ------------------------------------ +To suppress auto-opening the browser:: -Backend: + weightslab start --no-browser -- Ensure Weightslab serves gRPC and listens on host ``0.0.0.0:50051``. -- Enable backend TLS and client-auth for Envoy by setting: +Ports +----- - - ``GRPC_TLS_ENABLED=1`` - - ``GRPC_TLS_REQUIRE_CLIENT_AUTH=1`` - - optional ``GRPC_TLS_CERT_DIR=~/certs`` to change the default certificate directory - - ``GRPC_TLS_CERT_FILE`` / ``GRPC_TLS_KEY_FILE`` / ``GRPC_TLS_CA_FILE`` -- Optionally set ``GRPC_AUTH_TOKEN`` (or ``GRPC_AUTH_TOKENS``) to enforce - metadata token authentication in addition to mTLS. +- UI HTTP server: ``8080`` by default (``--port PORT`` or ``$WEIGHTSLAB_UI_PORT``) +- Backend gRPC: ``50051`` by default (``--backend-port PORT`` or ``$GRPC_BACKEND_PORT``) -If the per-file ``GRPC_TLS_*_FILE`` variables are not set, WeightsLab defaults to -``~/certs/backend-server.crt``, ``~/certs/backend-server.key``, and ``~/certs/ca.crt``. -If config/hyperparameters define TLS keys (``grpc_tls_*``), those values are used -before environment variables. +If port ``8080`` is already in use, ``weightslab start`` automatically finds the +next free port and logs the one it chose. -Envoy: +Secure mode (HTTPS + mTLS) +-------------------------- -- Ensure ``envoy.yaml`` cluster points to host backend: - ``host.docker.internal:50051``. -- Ensure cert files exist at ``../weights_studio/envoy/certs``: +The default is plain HTTP (no cert files required, easiest for local dev). +To enable HTTPS between the browser and the UI server, and mTLS between the +UI server and the backend: - - ``envoy-server.crt`` / ``envoy-server.key`` (browser->Envoy TLS) - - ``envoy-client.crt`` / ``envoy-client.key`` and ``ca.crt`` (Envoy->backend mTLS) +1. Generate TLS certificates once:: -Frontend: + weightslab se -- Ensure frontend points to Envoy (default ``https://localhost:8080``). + Certificates are placed in ``~/.weightslab-certs`` + (or ``$WEIGHTSLAB_CERTS_DIR``). + Follow the printed instructions to export ``WEIGHTSLAB_CERTS_DIR`` globally. -Sanity checks: +2. Start the UI in secure mode:: -- Studio reachable at ``https://localhost:5173``. -- Envoy endpoint reachable at ``https://localhost:8080``. -- Envoy admin is private by default (loopback inside container). + weightslab start --certs -Agent Usage in Weights Studio ------------------------------ + ``--certs`` reads ``$WEIGHTSLAB_CERTS_DIR`` (single source of truth) and: -Weights Studio includes an agent bar and an expandable agent history window. -The agent can run with either: + - Serves HTTPS using ``ui-server.crt`` / ``ui-server.key`` + - Presents ``ui-client.crt`` / ``ui-client.key`` to the backend (mTLS) + - Expects the backend CA at ``ca.crt`` -- a local Ollama provider configured on the backend -- a cloud OpenRouter provider configured at startup or initialized from the UI +3. Configure the backend to require mTLS:: -Local Ollama workflow -~~~~~~~~~~~~~~~~~~~~~ + export GRPC_TLS_ENABLED=1 + export GRPC_TLS_REQUIRE_CLIENT_AUTH=1 + export WEIGHTSLAB_CERTS_DIR=~/.weightslab-certs -If the backend is configured with ``provider: ollama`` and the Ollama server is -running, the agent is available immediately after backend startup. +Certificate files (all in ``$WEIGHTSLAB_CERTS_DIR``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Typical local setup: ++----------------------------+--------------------------------------------+ +| File | Purpose | ++============================+============================================+ +| ``ca.crt`` | CA certificate (trusted by all parties) | ++----------------------------+--------------------------------------------+ +| ``ui-server.crt/.key`` | UI server TLS cert (browser to server) | ++----------------------------+--------------------------------------------+ +| ``ui-client.crt/.key`` | UI client mTLS cert (server to backend) | ++----------------------------+--------------------------------------------+ +| ``backend-server.crt/.key``| Backend gRPC TLS cert (loaded by backend) | ++----------------------------+--------------------------------------------+ +| ``.grpc_auth_token`` | Optional token for gRPC metadata auth | ++----------------------------+--------------------------------------------+ -1. Start Ollama. -2. Start WeightsLab. -3. Open Weights Studio. -4. Ask questions directly in the agent bar. +Regenerate certificates at any time with ``weightslab se --force-certs``. -Cloud OpenRouter workflow -~~~~~~~~~~~~~~~~~~~~~~~~~ - -If the backend is not initialized with a cloud key yet, Weights Studio shows -the agent as unconfigured and the input placeholder instructs the user to type -``/init``. - -``/init`` flow: - -1. Type ``/init`` in the agent input. -2. Choose manual API key entry or the OpenRouter OAuth flow. -3. Select a model from the available model list. -4. Confirm to initialize the runtime connection. - -The default cloud model is ``~google/gemini-flash-latest``. - -Available agent commands -~~~~~~~~~~~~~~~~~~~~~~~~ +Configuration reference +----------------------- -The agent bar supports these commands: +Backend environment variables (set before starting ``wl.serve()``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------+-------------------------+----------------------------------------------------+ +| Variable | Default | Description | ++==================================+=========================+====================================================+ +| ``WEIGHTSLAB_LOG_LEVEL`` | ``INFO`` | Log level (``DEBUG``, ``INFO``, ...) | ++----------------------------------+-------------------------+----------------------------------------------------+ +| ``GRPC_BACKEND_HOST`` | ``0.0.0.0`` | Host the backend gRPC server binds to | ++----------------------------------+-------------------------+----------------------------------------------------+ +| ``GRPC_BACKEND_PORT`` | ``50051`` | Port the backend gRPC server listens on | ++----------------------------------+-------------------------+----------------------------------------------------+ +| ``GRPC_TLS_ENABLED`` | ``0`` | ``1`` = enable TLS on the gRPC socket | ++----------------------------------+-------------------------+----------------------------------------------------+ +| ``GRPC_TLS_REQUIRE_CLIENT_AUTH`` | ``0`` | ``1`` = require client mTLS certificate | ++----------------------------------+-------------------------+----------------------------------------------------+ +| ``WEIGHTSLAB_CERTS_DIR`` | ``~/.weightslab-certs`` | Directory containing cert/key files | ++----------------------------------+-------------------------+----------------------------------------------------+ +| ``GRPC_AUTH_TOKEN`` | *(unset)* | Optional metadata-token auth (on top of mTLS) | ++----------------------------------+-------------------------+----------------------------------------------------+ +| ``GRPC_MAX_MESSAGE_BYTES`` | ``268435456`` | Raise for large tensors / image batches | ++----------------------------------+-------------------------+----------------------------------------------------+ +| ``WEIGHTSLAB_DISABLE_WATCHDOGS`` | ``0`` | ``1`` = disable watchdogs (use with breakpoints) | ++----------------------------------+-------------------------+----------------------------------------------------+ + +UI server environment variables (set before ``weightslab start``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++---------------------------+-------------------------+--------------------------------------------------+ +| Variable | Default | Description | ++===========================+=========================+==================================================+ +| ``WEIGHTSLAB_UI_HOST`` | ``0.0.0.0`` | Interface the UI server binds to | ++---------------------------+-------------------------+--------------------------------------------------+ +| ``WEIGHTSLAB_UI_PORT`` | ``8080`` | HTTP port (``--port`` flag overrides) | ++---------------------------+-------------------------+--------------------------------------------------+ +| ``GRPC_BACKEND_HOST`` | ``localhost`` | Backend gRPC host to proxy to | ++---------------------------+-------------------------+--------------------------------------------------+ +| ``GRPC_BACKEND_PORT`` | ``50051`` | Backend gRPC port to proxy to | ++---------------------------+-------------------------+--------------------------------------------------+ +| ``WEIGHTSLAB_CERTS_DIR`` | ``~/.weightslab-certs`` | Certs dir (read when ``--certs``) | ++---------------------------+-------------------------+--------------------------------------------------+ + +Frontend runtime feature toggles +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These are injected as ``window.*`` globals when the UI is served. +Set them as environment variables before ``weightslab start``. + ++--------------------------------------+----------+----------------------------------------------------+ +| Variable | Default | Effect when ``0`` / ``false`` | ++======================================+==========+====================================================+ +| ``ENABLE_PLOTS`` | ``1`` | Remove plots board + Signals card | ++--------------------------------------+----------+----------------------------------------------------+ +| ``ENABLE_DATA_EXPLORATION`` | ``1`` | Remove data grid + metadata/details panel | ++--------------------------------------+----------+----------------------------------------------------+ +| ``ENABLE_HYPERPARAMETERS_OPTIMIZATION`` | ``1`` | Remove Hyperparameters section (read-only HPs) | ++--------------------------------------+----------+----------------------------------------------------+ +| ``ENABLE_AGENT`` | ``1`` | Remove agent chat bar | ++--------------------------------------+----------+----------------------------------------------------+ +| ``WS_HISTOGRAM_MAX_BINS`` | ``512`` | Cap on metadata histogram bars | ++--------------------------------------+----------+----------------------------------------------------+ +| ``BB_THUMB_RENDER`` | ``10`` | Max bounding boxes per thumbnail (per overlay) | ++--------------------------------------+----------+----------------------------------------------------+ +| ``BB_MODAL_RENDER`` | ``100`` | Max bounding boxes per modal image (per overlay) | ++--------------------------------------+----------+----------------------------------------------------+ + +Tunnel (remote backend) +----------------------- -- ``/init`` initializes OpenRouter from the UI -- ``/model`` opens the model chooser to switch the active OpenRouter model -- ``/reset`` clears the current agent runtime connection and status +If your backend is running remotely (e.g. a Colab notebook behind ``ngrok`` or +``bore``), forward it to a local port with:: -History behavior -~~~~~~~~~~~~~~~~ + weightslab tunnel bore.pub:12345 -- Command entries such as ``/init``, ``/model``, and ``/reset`` are shown on - the user side of the history. -- Agent lifecycle events such as connection setup, model changes, and reset - events are shown as separate log-style entries. -- A pinned instruction line at the top of the history summarizes the available - commands and shows the full instruction text on hover. +Then ``weightslab start`` on the same machine proxies to it as if local. +The tunnel is raw TCP — the backend must be plaintext (``GRPC_TLS_ENABLED=0``). Agent Usage in Weights Studio ------------------------------ +------------------------------ Weights Studio includes an agent bar and an expandable agent history window. The agent can run with either: @@ -205,9 +209,9 @@ running, the agent is available immediately after backend startup. Typical local setup: 1. Start Ollama. -2. Start WeightsLab. -3. Open Weights Studio. -4. Ask questions directly in the agent bar. +2. Start WeightsLab (``wl.serve(serving_grpc=True)``). +3. Start Weights Studio (``weightslab start``). +4. Ask questions in the agent bar. Cloud OpenRouter workflow ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -228,215 +232,88 @@ The default cloud model is ``~google/gemini-flash-latest``. Available agent commands ~~~~~~~~~~~~~~~~~~~~~~~~ -The agent bar supports these commands: - -- ``/init`` initializes OpenRouter from the UI -- ``/model`` opens the model chooser to switch the active OpenRouter model -- ``/reset`` clears the current agent runtime connection and status +- ``/init`` — initialize OpenRouter from the UI +- ``/model`` — open the model chooser to switch the active OpenRouter model +- ``/reset`` — clear the current agent runtime connection and status History behavior ~~~~~~~~~~~~~~~~ - Command entries such as ``/init``, ``/model``, and ``/reset`` are shown on the user side of the history. -- Agent lifecycle events such as connection setup, model changes, and reset - events are shown as separate log-style entries. -- A pinned instruction line at the top of the history summarizes the available - commands and shows the full instruction text on hover. +- Agent lifecycle events (connection setup, model changes, reset) are shown as + separate log-style entries. +- A pinned instruction line at the top summarizes the available commands. -Server integration (AWS example) --------------------------------- +Bundled examples +---------------- -This section describes a practical cloud deployment path for Weightslab + -Weights Studio on AWS. +Run a bundled example in one command (installs its requirements automatically):: -Recommended production architecture -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + weightslab start example # classification (default) + weightslab start example --seg # segmentation + weightslab start example --det # detection + weightslab start example --3d_det # 3D LiDAR point-cloud detection -- **UI**: Weights Studio frontend behind HTTPS (ALB). -- **gRPC-Web bridge**: Envoy service reachable by frontend. -- **Backend**: Weightslab training service (Python gRPC). -- **Storage**: EBS/EFS/S3 for logs/checkpoints depending on your workflow. +In another terminal, start the UI:: -Two common deployment options -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + weightslab start -1. **EC2 (fastest to start)** +See ``weightslab start example --help`` for all options. - - Run Weightslab backend process on the VM (port ``50051``). - - Run Envoy + frontend via Docker Compose. - - Use one security group to expose only HTTPS (and optionally admin ports - privately). +Cloud deployment +---------------- -2. **ECS/Fargate (container-native)** +Because the UI is a plain Python process, cloud deployment is straightforward: - - Service A: frontend container. - - Service B: Envoy container. - - Service C: Weightslab backend container (or external training worker). - - Route through ALB + target groups. +1. Install WeightsLab on the server:: -Minimum port plan (AWS) -~~~~~~~~~~~~~~~~~~~~~~~ + pip install weightslab -- Public ingress: - - ``443`` (HTTPS to frontend/ALB) -- Internal service ports: - - ``8080`` (Envoy gRPC-Web listener) - - ``50051`` (Weightslab backend gRPC) - - ``9901`` (Envoy admin, keep private) - - ``5173`` (frontend dev port; avoid exposing directly in production) +2. Run ``weightslab se`` once to generate certificates. -Security group guidance -~~~~~~~~~~~~~~~~~~~~~~~ +3. Start the backend in your training process (``wl.serve(serving_grpc=True)``). -- Allow ``443`` from trusted client CIDRs (or internet if required). -- Restrict ``50051`` and ``8080`` to VPC/internal security groups. -- Restrict ``9901`` to private admin subnet/bastion only. -- Do not expose backend gRPC directly to the public internet. +4. Start the UI process:: -TLS and domains -~~~~~~~~~~~~~~~ + WEIGHTSLAB_UI_HOST=0.0.0.0 weightslab start --port 8080 --certs --no-browser -- Terminate TLS at ALB using ACM certificates. -- Use a DNS record (Route 53) for the studio hostname. -- Forward ALB target traffic to frontend service. -- Keep Envoy/backend internal where possible. +5. Put a reverse proxy (nginx / ALB / Caddy) in front of port ``8080`` and + expose only ``443`` publicly. -Environment mapping (cloud) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The UI and backend can run on different machines — set ``--backend-host`` and +``--backend-port`` accordingly. -For Weights Studio frontend + Envoy alignment, set environment variables -consistently with your deployed endpoints: +Example systemd unit +~~~~~~~~~~~~~~~~~~~~ .. code-block:: ini - # frontend resolves gRPC-web through Envoy - WS_SERVER_PROTOCOL=https - WS_SERVER_HOST=studio.your-domain.com - WS_SERVER_PORT=443 - VITE_HISTOGRAM_MAX_BINS=512 - BB_THUMB_RENDER=10 - BB_MODAL_RENDER=100 - - # envoy / backend internal wiring - ENVOY_PORT=8080 - ENVOY_ADMIN_PORT=9901 - GRPC_BACKEND_PORT=50051 - -If Envoy is internal-only and frontend is public, ensure frontend requests are -routed to the Envoy endpoint through your internal load-balancing design. - -AWS deployment checklist -~~~~~~~~~~~~~~~~~~~~~~~~ - -1. Provision VPC/subnets and security groups. -2. Deploy backend gRPC service and verify ``:50051`` internally. -3. Deploy Envoy and verify routing to backend. -4. Deploy frontend with correct ``WS_SERVER_*`` values. -5. Attach ALB + ACM certificate and configure HTTPS listener. -6. Validate UI actions (query, tagging, discard/restore, plots). -7. Add monitoring/logging (CloudWatch metrics/logs). - -Operational notes -~~~~~~~~~~~~~~~~~ - -- Prefer long-running backend workers for stable interactive sessions. -- Keep checkpoint/log storage durable (EBS/EFS/S3 strategy). -- If using autoscaling, ensure session and backend availability are handled - explicitly for active users. -- For multi-environment setups, keep per-environment ``.env`` templates - versioned and reviewed. - -Concrete EC2 + Docker Compose + systemd recipe -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Use this pattern for a simple single-VM production-like deployment. - -1. Provision EC2 - - - Ubuntu 22.04 (or similar), attached security group. - - Open only ``443`` publicly. - - Keep ``50051``, ``8080``, ``9901`` private (VPC/admin only). - -2. Install runtime dependencies - - - Docker Engine + Docker Compose plugin - - Python environment for your Weightslab backend process - -3. Configure environment - - In ``weights_studio/docker/.env`` (or environment management equivalent): - - .. code-block:: ini - - VITE_PORT=5173 - WS_SERVER_PROTOCOL=https - WS_SERVER_HOST=studio.your-domain.com - WS_SERVER_PORT=443 - VITE_HISTOGRAM_MAX_BINS=512 - BB_THUMB_RENDER=10 - BB_MODAL_RENDER=100 - - ENVOY_PORT=8080 - ENVOY_ADMIN_PORT=9901 - GRPC_BACKEND_PORT=50051 - -4. Start backend service - - Start Weightslab gRPC in your training/runtime process and ensure it binds - to a reachable interface (for example ``0.0.0.0:50051``). - -5. Start studio containers - - From ``weights_studio/docker``: - - .. code-block:: bash + [Unit] + Description=Weights Studio UI + After=network.target - docker compose up -d + [Service] + EnvironmentFile=/etc/weightslab/env + ExecStart=/usr/local/bin/weightslab start --port 8080 --no-browser + Restart=on-failure + RestartSec=5 -6. Add process supervision (systemd) + [Install] + WantedBy=multi-user.target - Use systemd to ensure services restart on reboot/failure. - - Example unit for studio compose stack: - - .. code-block:: ini - - [Unit] - Description=Weights Studio (Docker Compose) - Requires=docker.service - After=docker.service - - [Service] - Type=oneshot - WorkingDirectory=/opt/weights_studio/docker - ExecStart=/usr/bin/docker compose up -d - ExecStop=/usr/bin/docker compose down - RemainAfterExit=yes - TimeoutStartSec=0 - - [Install] - WantedBy=multi-user.target - - Enable and start: - - .. code-block:: bash - - sudo systemctl daemon-reload - sudo systemctl enable weights-studio - sudo systemctl start weights-studio - -7. Put HTTPS in front (ALB + ACM) +Building the frontend from source +---------------------------------- - - Attach ACM certificate to ALB listener on ``443``. - - Route DNS (Route 53) to ALB. - - Forward traffic to frontend target. +The pre-built SPA is vendored into ``weightslab/ui/static/``. To rebuild from +the ``weights_studio`` source repository and update the vendored copy:: -8. Validate end-to-end + # from the weights_studio repo + npm ci && npm run build - - Open studio URL. - - Verify sample query, tag/discard actions, and plot refresh. - - Verify backend logs and Envoy admin stats. + # from the weightslab repo + rm -rf weightslab/ui/static/* + cp -R ../weights_studio/dist/. weightslab/ui/static/ UI controls and actions ----------------------- @@ -446,50 +323,30 @@ Top header controls - **Dark mode toggle**: switch light/dark theme. - **Refresh button**: manually refresh dynamic stats in visible grid. -- **Refresh config popover**: - - Data auto-refresh enable/disable + interval - - Plot auto-refresh enable/disable + interval - - Clear cache and reload page -- **Training button** (Resume/Pause): toggles ``is_training`` via backend command. -- **Mode selector** (dropdown next to training button): - - ``train`` mode - - ``audit`` mode (sets ``auditorMode``) +- **Refresh config popover**: data/plot auto-refresh, clear cache. +- **Training button** (Resume/Pause): toggles ``is_training`` via backend. +- **Mode selector**: ``train`` mode / ``audit`` mode. Left panel ~~~~~~~~~~ -- **Training card**: - - Training state pill (running/paused/pending) - - Connection status text - - Live metrics and progress -- **Tags card**: - - Tag chips - - New tag input - - Painter toggle - - Add/remove painter mode switch -- **Details card**: - - Grid settings (cell size + resolution + apply) - - Segmentation overlays (Raw, GT, Pred, Diff, Split view) - - Split colors - - Metadata field toggles +- **Training card**: training state pill, connection status, live metrics. +- **Tags card**: tag chips, new tag input, painter toggle. +- **Details card**: grid settings, segmentation overlays, metadata field toggles. Grid interactions ~~~~~~~~~~~~~~~~~ - Drag selection rectangle (multi-select). - ``Ctrl`` multi-select support. -- Right-click context menu actions: - - Manage tags - - Remove all tags - - Discard selected samples - - Restore selected samples +- Right-click context menu: manage tags, discard/restore samples. The UI pauses training before data-modifying actions to keep edits safe. Bottom bar ~~~~~~~~~~ -- Batch slider for navigation over samples. +- Batch slider for sample navigation. - Start/end batch index labels. - Total and active sample counters. @@ -506,30 +363,18 @@ Signal plots Per-signal cards include: -- Reset zoom -- CSV export -- JSON export -- Settings (curve color, smoothing, std band, markers) +- Reset zoom, CSV/JSON export, settings (curve color, smoothing, std band, + markers). +- Right-click: reset zoom, change curve color, load weights at step, hide/show + curve, break by slices, copy/save chart image. -Right-click menu on plots includes: +WeightsLab CLI console +---------------------- -- Reset X/Y/all zoom -- Change curve color -- Load weights at clicked step -- Hide/unhide curve -- Break by slices -- Copy chart image -- Save chart image - -Weightslab CLI console (dev) ----------------------------- - -The Weightslab CLI console is a local developer REPL for inspecting and +The WeightsLab CLI console is a local developer REPL for inspecting and controlling a running experiment through the global ledger. -- Transport: local TCP text commands with JSON responses. -- Intended scope: development/debug only. -- Security model: localhost binding by default, plain-text protocol. +Transport: local TCP text commands with JSON responses. How to start it ~~~~~~~~~~~~~~~ @@ -543,70 +388,38 @@ From your training script (recommended): wl.serve(serving_grpc=True, serving_cli=True) wl.keep_serving() -Standalone server: - -.. code-block:: bash +Connect from a terminal:: - python -m weightslab.backend.cli serve --host localhost --port 60000 + weightslab cli # auto-discover port + weightslab cli --port 60000 # or specify one -Connect a client manually: - -.. code-block:: bash - - python -m weightslab.backend.cli client --host localhost --port 60000 - -If no port is provided (or port is ``0``), the server picks a free port. - -Console actions and commands -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Console actions +~~~~~~~~~~~~~~~ -Full command-by-command reference (discovery/help, training control, registry -inspection, sample-level dataset operations, hyperparameters, evaluation, -audit mode, the AI agent, and session control) — with syntax, flags, and -worked examples for every one — lives in :doc:`user_commands`. Quick -summary of what's available: +Full reference: :doc:`user_commands`. Quick summary: - Discovery/help: ``help``, ``status``, ``dump``, ``ledger_dump``. - Training control: ``pause`` / ``resume``. - Registry inspection: ``list_models``, ``list_optimizers``, ``list_loaders``, ``plot_model [model_name]``. -- Sample-level dataset operations: ``list_uids``, ``discard``, ``undiscard``, +- Sample-level operations: ``list_uids``, ``discard``, ``undiscard``, ``add_tag``. - Hyperparameters: ``hp``, ``set_hp``. - Evaluation: ``evaluate``, ``eval_status``, ``cancel_eval``. - Audit mode: ``audit [on|off]``. -- AI agent: ``agent ...`` / ``query`` / ``ask`` — see :doc:`agent`. +- AI agent: ``agent`` / ``query`` / ``ask`` — see :doc:`agent`. - Session control: ``exit`` / ``quit``, ``clear`` / ``cls``. -Developer notes -~~~~~~~~~~~~~~~ - -- Prefer CLI for quick diagnosis and manual interventions. -- Keep CLI port private (localhost or private subnet only). -- Use Weights Studio for richer visual workflows; use CLI for low-latency - command-driven operations. - -Weightslab workflow recommendation ----------------------------------- - -Typical loop for productive usage: - -1. Start Weightslab backend and studio stack. -2. Monitor training metrics and sample-level signals. -3. Use grid metadata + modal details to inspect hard or noisy samples. -4. Tag slices (e.g., outliers, hard cases). -5. Discard low-value samples and continue training. -6. Use train/audit mode toggles to inspect safely without weight updates. -7. Use plot controls to inspect branch transitions and checkpoint behavior. - Troubleshooting --------------- -- Studio loads but no data: - check backend gRPC is running and Envoy target is reachable. -- Connection/reset errors: - verify ``ENVOY_PORT`` and backend port mapping. -- Wrong endpoint: - verify ``WS_SERVER_HOST/PORT/PROTOCOL`` in docker environment. -- No plot updates: - verify plot auto-refresh setting and backend logger data availability. +- **Studio loads but no data**: check backend gRPC is running on the expected + port (``--backend-port``) and that there is no firewall blocking the + connection. +- **Port conflict**: ``weightslab start`` auto-selects the next free port and + logs it; or pass ``--port PORT`` to pick a specific one. +- **No plot updates**: check plot auto-refresh setting and backend logger data. +- **TLS errors with --certs**: run ``weightslab se`` first to generate certs, + then export ``WEIGHTSLAB_CERTS_DIR``. +- **Connection refused on remote backend**: use ``weightslab tunnel`` to forward + the remote port locally. diff --git a/pyproject.toml b/pyproject.toml index f4ab6ec9..c67657b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,9 +28,16 @@ classifiers = [ # - Prefer compatible ranges, then pin exact versions only in CI/lockfiles. dependencies = [ # Core data / persistence - "numpy>=1.25.2,<2.0; python_version < '3.13'", - "numpy>=2.1,<3; python_version >= '3.13'", - "pandas>=2.2.3,<3", + # numpy 2 is supported: weightslab's source uses no numpy-1-only APIs, and its + # compiled deps (torch, pandas, h5py, tables, onnx) ship numpy-2 wheels at the + # versions pip resolves. Range left open (<3) so Colab's numpy 2.x installs + # without a downgrade. Verified: import + torch/pandas interop + gRPC on np 2.4. + "numpy>=1.24,<3", + # Floor at 2.2.2 (not 2.2.3) so Colab's pinned pandas==2.2.2 is accepted + # as-is: google-colab 1.0.0 requires pandas==2.2.2, and a >=2.2.3 floor forced + # an upgrade to 2.3.x that breaks that pin. Clean installs still pick the + # newest numpy-2-safe pandas (2.3.x). + "pandas>=2.2.2,<3", "duckdb>=1.1,<2", # signal/sample/instance history store "PyYAML>=6.0.3,<7", "dill>=0.3.8,<0.5", @@ -39,16 +46,25 @@ dependencies = [ "xxhash>=3.4,<4.1", "tables>=3.9,<4", # h5 deps - # PyTorch stack - "torch>=2.1,<2.9; python_version < '3.13'", - "torch==2.9; python_version >= '3.13'", - "torchvision>=0.16,<0.24; python_version < '3.13'", - "torchvision>=0.24,<1; python_version >= '3.13'", + # PyTorch stack. Upper bounds left open (<3 / <1) so a pre-installed modern + # torch (e.g. Colab's torch 2.9 / torchvision 0.24) is accepted as-is instead + # of being downgraded — the reinstall churn is what dragged numpy across the + # 1.x/2.x boundary and broke the ABI. Floors kept; pip picks a compatible + # numpy-2 build on a clean install. + "torch>=2.1,<=2.9", + "torchvision>=0.16,<1", "torchmetrics>=1.9", # Serving / schema "grpcio>=1.80,<2", - "protobuf>=4.25,<7", + # Floor tracks the checked-in gencode: the *_pb2.py are regenerated with + # grpcio-tools ~=1.68 (protoc gencode 5.28.1), so any runtime >=5.28.1 satisfies + # protobuf's runtime>=gencode rule — including Colab's stock protobuf 5.29.6, so + # NO upgrade is forced on Colab. IMPORTANT: regenerate the stubs with + # grpcio-tools ~=1.68 to keep the gencode Colab-compatible; a newer grpcio-tools + # bumps the gencode and reintroduces the VersionError on Colab. Upper bound <8 + # keeps modern runtimes (7.x) working. + "protobuf>=5.28.1,<8", "pydantic>=2.7,<3", # Imaging @@ -99,15 +115,16 @@ utest = [ ] [project.scripts] -weightslab = "weightslab.ui_docker_bridge:main" +weightslab = "weightslab.cli:main" [project.urls] Homepage = "https://github.com/GrayboxTech/weightslab" [tool.setuptools] -# Ship non-.py assets that live inside the package (docker stack + bundled -# examples). Combined with the explicit globs below so the CLI subcommands -# (`ui launch`, `start example`) find their files in a pip-installed package. +# Ship non-.py assets that live inside the package (cert scripts, bundled +# examples, pre-built UI). Combined with the explicit globs below so the CLI +# subcommands (`start`, `start example`, `se`) find their files in a +# pip-installed package. include-package-data = true [tool.setuptools.packages.find] @@ -115,18 +132,19 @@ where = ["."] include = ["weightslab", "weightslab.*"] [tool.setuptools.package-data] -# Bundle the docker stack (compose files, envoy/nginx configs, shell scripts) -# and the example projects (main.py, config.yaml, sample data) that the -# `weightslab` CLI resolves relative to the installed package. +# Bundle the cert-generation scripts, the example projects (main.py, config.yaml, +# sample data), and the pre-built Weights Studio SPA that `weightslab start` +# serves directly — no Docker required. weightslab = [ - "docker/**/*", + "ui/utils/**/*", "examples/**/*", + # Pre-built Weights Studio SPA served by `weightslab start`. + "ui/static/**/*", ] [tool.setuptools.exclude-package-data] # Never ship generated/local artifacts even if present on disk at build time. weightslab = [ - "docker/docker/.env", "**/__pycache__/**", "**/*.pyc", ] diff --git a/tests/backend/test_cli.py b/tests/backend/test_cli.py new file mode 100644 index 00000000..543ea5c0 --- /dev/null +++ b/tests/backend/test_cli.py @@ -0,0 +1,274 @@ +"""Tests for the Docker-free WeightsLab CLI (weightslab.cli). + +Covers the command set that survived the Docker removal: se (secure env), +start (native UI), start example, cli, tunnel, banner/help — plus cert +generation and the small path/script helpers. There is no Docker, Envoy, or +compose code left to test. +""" +import argparse +import contextlib +import io +import os +import sys +import unittest +from pathlib import Path +from unittest.mock import patch, MagicMock + +from weightslab.cli import ( + _convert_to_git_bash_path, + _ensure_scripts_executable, + _generate_certs_with_fallback, + _get_cert_script, + _get_example_dir, + _install_example_requirements, + _make_executable, + example_start, + main, + ui_secure_environment, + ui_start_native, +) + + +class TestScriptsExecutable(unittest.TestCase): + """Bundled .sh scripts are made executable so users skip the manual `chmod +x`.""" + + @unittest.skipIf(sys.platform == "win32", "execute bit is POSIX-only") + def test_make_executable_adds_exec_bits(self): + import stat as _stat + import tempfile + with tempfile.NamedTemporaryFile(suffix=".sh", delete=False) as f: + path = f.name + try: + os.chmod(path, 0o644) + _make_executable(path) + mode = os.stat(path).st_mode + self.assertTrue(mode & _stat.S_IXUSR) + self.assertTrue(mode & _stat.S_IXGRP) + self.assertTrue(mode & _stat.S_IXOTH) + finally: + os.unlink(path) + + def test_make_executable_is_noop_on_windows(self): + with patch("weightslab.cli._is_windows", return_value=True): + with patch("weightslab.cli.os.chmod") as mock_chmod: + _make_executable("/whatever/path.sh") + mock_chmod.assert_not_called() + + def test_make_executable_swallows_oserror(self): + with patch("weightslab.cli.os.stat", side_effect=OSError("denied")): + _make_executable("/root/owned.sh") # should not raise + + def test_ensure_scripts_executable_noop_on_windows(self): + with patch("weightslab.cli._is_windows", return_value=True): + with patch("weightslab.cli._make_executable") as mock_mk: + _ensure_scripts_executable() + mock_mk.assert_not_called() + + def test_cert_script_is_bundled_under_ui_utils(self): + cert_script = _get_cert_script() + self.assertTrue(str(cert_script).replace("\\", "/").endswith( + "weightslab/ui/utils/generate-certs-auth-token.sh"), cert_script) + + +class TestPathConversion(unittest.TestCase): + def test_windows_path_conversion(self): + self.assertEqual( + _convert_to_git_bash_path(r"C:\Users\testuser\.weightslab-certs"), + "/mnt/c/Users/testuser/.weightslab-certs", + ) + + def test_unix_path_passthrough(self): + self.assertEqual( + _convert_to_git_bash_path("/home/testuser/.weightslab-certs"), + "/home/testuser/.weightslab-certs", + ) + + +class TestCertGeneration(unittest.TestCase): + @patch("weightslab.cli._run_shell_script", return_value=0) + def test_generate_certs_forwards_certs_dir(self, mock_shell): + rc = _generate_certs_with_fallback(force_certs=False, certs_dir="/custom/certs") + self.assertEqual(rc, 0) + env_vars = mock_shell.call_args.args[2] + self.assertIsInstance(env_vars, dict) + self.assertIn("WEIGHTSLAB_CERTS_DIR", env_vars) + self.assertIn("custom/certs", env_vars["WEIGHTSLAB_CERTS_DIR"].replace("\\", "/")) + + @patch("weightslab.cli._run_shell_script", return_value=0) + def test_generate_certs_without_dir_passes_no_env(self, mock_shell): + _generate_certs_with_fallback(force_certs=False) + self.assertIsNone(mock_shell.call_args.args[2]) + + +class TestUiSecureEnvironment(unittest.TestCase): + @patch("weightslab.cli.CertAuthManager") + @patch("weightslab.cli._generate_certs_with_fallback", return_value=0) + def test_ui_secure_environment_success(self, mock_gen_certs, mock_cert_manager): + """`weightslab se`: generate certs + token, export WEIGHTSLAB_CERTS_DIR.""" + mgr = MagicMock() + mgr.get_or_create_auth_token.return_value = "fake_token" + mock_cert_manager.return_value = mgr + with self.assertLogs("weightslab.cli", level="INFO") as log_context: + ui_secure_environment(argparse.Namespace(force_certs=False)) + self.assertTrue(any("Certificates generated successfully" in m for m in log_context.output)) + mock_gen_certs.assert_called_once_with(force_certs=False, certs_dir=mgr.certs_dir) + mgr.certs_dir.mkdir.assert_called_once() + self.assertTrue(any("WEIGHTSLAB_CERTS_DIR exported" in m for m in log_context.output)) + + @patch("weightslab.cli._generate_certs_with_fallback", return_value=0) + def test_ui_secure_environment_force_certs(self, mock_gen_certs): + with patch.dict(os.environ, {}, clear=False): + ui_secure_environment(argparse.Namespace(force_certs=True)) + self.assertTrue(mock_gen_certs.call_args.kwargs["force_certs"]) + + @patch("weightslab.cli._generate_certs_with_fallback", return_value=1) + def test_ui_secure_environment_cert_failure(self, _mock_gen_certs): + with self.assertRaises(SystemExit) as ctx: + ui_secure_environment(argparse.Namespace(force_certs=False)) + self.assertEqual(ctx.exception.code, 1) + + +class TestUiStartNative(unittest.TestCase): + """`weightslab start` serves the bundled SPA + gRPC-Web proxy (no Docker).""" + + @patch("weightslab.ui.server.serve_ui") + def test_start_invokes_serve_ui_with_defaults(self, mock_serve): + with patch.dict(os.environ, {}, clear=False): + for k in ("WEIGHTSLAB_UI_HOST", "WEIGHTSLAB_UI_PORT", + "GRPC_BACKEND_HOST", "GRPC_BACKEND_PORT"): + os.environ.pop(k, None) + args = argparse.Namespace(port=9123, host=None, backend_host=None, + backend_port=None, no_browser=True, certs=False) + ui_start_native(args) + mock_serve.assert_called_once() + kwargs = mock_serve.call_args.kwargs + self.assertEqual(kwargs["backend_port"], 50051) + self.assertFalse(kwargs["open_browser"]) + self.assertIsNone(kwargs["certs_dir"]) # no --certs -> unsecured + + @patch("weightslab.ui.server.serve_ui") + @patch("weightslab.cli.CertAuthManager") + def test_start_certs_without_valid_certs_falls_back_to_http(self, mock_mgr, mock_serve): + mgr = MagicMock() + mgr.has_valid_certs.return_value = False + mock_mgr.from_env_or_default.return_value = mgr + args = argparse.Namespace(port=9124, host=None, backend_host=None, + backend_port=None, no_browser=True, certs=True) + with self.assertLogs("weightslab.cli", level="WARNING"): + ui_start_native(args) + self.assertIsNone(mock_serve.call_args.kwargs["certs_dir"]) + + +class TestExampleStart(unittest.TestCase): + @patch("weightslab.cli.subprocess.run") + def test_example_start_runs_classification(self, mock_run): + mock_run.return_value = MagicMock(returncode=0) + with self.assertLogs("weightslab.cli", level="INFO") as log_context: + example_start(argparse.Namespace()) + cmd = mock_run.call_args.args[0] + self.assertEqual(cmd[0], sys.executable) + main_py = cmd[1].replace("\\", "/") + self.assertTrue(main_py.endswith("examples/PyTorch/wl-classification/main.py"), main_py) + self.assertTrue(any("classification (cls) example" in m for m in log_context.output)) + + @patch("weightslab.cli.subprocess.run") + def test_example_start_propagates_nonzero_exit(self, mock_run): + mock_run.return_value = MagicMock(returncode=3) + with self.assertRaises(SystemExit) as ctx: + example_start(argparse.Namespace()) + self.assertEqual(ctx.exception.code, 3) + + @patch("weightslab.cli._get_example_dir", return_value=Path("/does/not/exist")) + def test_example_start_errors_when_missing(self, _mock_dir): + with self.assertRaises(SystemExit) as ctx: + example_start(argparse.Namespace()) + self.assertEqual(ctx.exception.code, 1) + + def test_example_dir_points_at_bundled_example(self): + self.assertTrue((_get_example_dir("wl-classification") / "main.py").exists()) + + @patch("weightslab.cli.subprocess.run") + def test_example_start_seg_runs_segmentation(self, mock_run): + mock_run.return_value = MagicMock(returncode=0) + example_start(argparse.Namespace(example_kind="seg")) + main_py = mock_run.call_args.args[0][1].replace("\\", "/") + self.assertTrue(main_py.endswith("examples/PyTorch/wl-segmentation/main.py"), main_py) + + +class TestInstallExampleRequirements(unittest.TestCase): + @patch("weightslab.cli.subprocess.run") + def test_installs_requirements_non_interactively_when_present(self, mock_run): + import tempfile + mock_run.return_value = MagicMock(returncode=0) + with tempfile.TemporaryDirectory() as tmp: + (Path(tmp) / "requirements.txt").write_text("numpy\n") + _install_example_requirements(Path(tmp)) + cmd = mock_run.call_args.args[0] + self.assertEqual(cmd[:5], [sys.executable, "-m", "pip", "install", "-r"]) + self.assertIn("--no-input", cmd) + + @patch("weightslab.cli.subprocess.run") + def test_skips_when_no_requirements_file(self, mock_run): + import tempfile + with tempfile.TemporaryDirectory() as tmp: + _install_example_requirements(Path(tmp)) + mock_run.assert_not_called() + + +class TestMainCLI(unittest.TestCase): + """The CLI exposes exactly: se, start (+example), cli, tunnel, help. No `ui`.""" + + @patch("weightslab.cli.ui_secure_environment") + def test_main_dispatches_se(self, mock_se): + with patch("sys.argv", ["weightslab", "se"]): + main() + mock_se.assert_called_once() + + @patch("weightslab.cli.ui_start_native") + def test_main_bare_start_launches_native_ui(self, mock_start): + with patch("sys.argv", ["weightslab", "start"]): + main() + mock_start.assert_called_once() + + @patch("weightslab.cli.example_start") + def test_main_dispatches_start_example(self, mock_example): + with patch("sys.argv", ["weightslab", "start", "example"]): + main() + mock_example.assert_called_once() + + def test_main_ui_command_is_gone(self): + # `ui` is no longer a valid subcommand (Docker launcher removed). + with patch("sys.argv", ["weightslab", "ui", "launch"]): + with self.assertRaises(SystemExit): + main() + + def test_main_help_does_not_crash(self): + with patch("sys.argv", ["weightslab", "help"]): + main() + + def test_main_no_args_does_not_crash(self): + with patch("sys.argv", ["weightslab"]): + main() + + +class TestBannerAndHelp(unittest.TestCase): + @staticmethod + def _capture_main(argv): + buf = io.StringIO() + with patch("sys.argv", argv): + with contextlib.redirect_stdout(buf): + try: + main() + except SystemExit: + pass + return buf.getvalue() + + def test_help_shows_new_command_set(self): + out = self._capture_main(["weightslab", "help"]) + self.assertIn("se", out) + self.assertIn("start", out) + self.assertIn("start example", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/backend/test_data_loader_interface.py b/tests/backend/test_data_loader_interface.py index 4da94914..000e45c9 100644 --- a/tests/backend/test_data_loader_interface.py +++ b/tests/backend/test_data_loader_interface.py @@ -353,6 +353,35 @@ def test_multiple_sequential_epochs_with_auto_reset(self): f"Each epoch should yield the same batch count, got {epoch_counts}", ) + def test_reset_is_callable_through_ledger_proxy(self): + """`reset()` must exist and be callable via the ledger Proxy handle. + + Ultralytics' trainer calls ``train_loader.reset()`` after closing mosaic + augmentation. The loader handed to it is a ledger Proxy wrapping this + interface, so a real ``reset()`` method (mapped to ``_reset_iterator``) + must be reachable through the proxy — otherwise the call resolves to a + non-callable None. Regression for the Ultralytics YOLO notebook crash + ``TypeError: 'NoneType' object is not callable``. + """ + DataLoaderInterface( + self.train_ds, batch_size=self.batch_size, + loader_name="reset_proxy_loader", register=True, compute_hash=True, + ) + loader = ledgers.get_dataloader("reset_proxy_loader") + + self.assertTrue(hasattr(loader, "reset")) + self.assertTrue(callable(loader.reset)) + + # A full epoch before reset. + batches_before = sum(1 for _ in loader) + self.assertGreater(batches_before, 0) + + # reset() must not raise, and the loader stays fully usable afterwards + # (a fresh full epoch with the same batch count can be iterated). + loader.reset() + batches_after = sum(1 for _ in loader) + self.assertEqual(batches_after, batches_before) + class TestDataLoaderReproducibility(unittest.TestCase): """Test RNG and iteration state reproducibility for dataloaders.""" diff --git a/tests/backend/test_ledgers.py b/tests/backend/test_ledgers.py index 8db41501..513b1caf 100644 --- a/tests/backend/test_ledgers.py +++ b/tests/backend/test_ledgers.py @@ -672,6 +672,49 @@ def __init__(self): delattr(proxy, "flag") self.assertFalse(hasattr(wrapped, "flag")) + def test_proxy_missing_attr_raises_attributeerror(self): + """Missing attributes on the wrapped object must raise AttributeError. + + Regression: __getattr__ used to `return None`, which made + `hasattr(proxy, missing)` report True and silently resolved + `proxy.missing` to a non-callable None. That broke capability probes + such as Ultralytics' `if not hasattr(loader, "reset")` shim, leaving + `loader.reset` as None -> `'NoneType' object is not callable`. + """ + proxy = Proxy(Dummy("x")) + + # Existing attribute still forwards. + self.assertEqual(proxy.name, "x") + + # Missing attribute raises (does not return None). + with self.assertRaises(AttributeError): + _ = proxy.definitely_not_here + + # hasattr therefore reports False, and getattr honors the default. + self.assertFalse(hasattr(proxy, "definitely_not_here")) + self.assertIsNone(getattr(proxy, "definitely_not_here", None)) + self.assertEqual(getattr(proxy, "definitely_not_here", 42), 42) + + def test_proxy_forwards_callable_capability(self): + """A method present on the wrapped object is discoverable and callable. + + Mirrors how a ledger Proxy wraps a DataLoaderInterface whose real + `reset()` must be reachable through the proxy. + """ + class _HasReset: + def __init__(self): + self.calls = 0 + + def reset(self): + self.calls += 1 + + wrapped = _HasReset() + proxy = Proxy(wrapped) + + self.assertTrue(hasattr(proxy, "reset")) + self.assertTrue(callable(proxy.reset)) + proxy.reset() + self.assertEqual(wrapped.calls, 1) if __name__ == "__main__": diff --git a/tests/backend/test_logger_core.py b/tests/backend/test_logger_core.py index eb44d569..291a66fc 100644 --- a/tests/backend/test_logger_core.py +++ b/tests/backend/test_logger_core.py @@ -16,8 +16,10 @@ - load_signal_history (dict format, list format) """ +import os +import time import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from weightslab.backend.logger import LoggerQueue @@ -834,5 +836,184 @@ def test_no_match_returns_none(self): self.assertIsNone(lg.resolve_graph_name("accuracy")) +class TestBackgroundFlushAndLossShapeAutotag(unittest.TestCase): + """LoggerQueue starts a background thread that periodically flushes staged + rows and auto-classifies every flag="loss" signal — off the caller's + thread, with zero setup required (see auto_loss_shape_signal_names). + + The actual tagging logic (_autotag_loss_shapes) is called directly here + rather than via time.sleep()-and-poll on the real background thread: with + many LoggerQueue instances created across this test file, a still-winding- + down daemon thread from an earlier test could otherwise tick during a + later test's `patch(...)` window and pollute its mock's call list. Calling + the method directly is deterministic and exercises the identical code path + the thread calls on every tick; test_background_flush_thread_starts_and_stops + separately covers that the thread itself starts/stops correctly. + """ + + def _lg_no_autoflush(self): + """A LoggerQueue with the background thread stopped immediately, so + only the explicit _autotag_loss_shapes() calls below can tag anything.""" + lg = _lg() + lg.stop_background_flush() + return lg + + def test_background_flush_thread_starts_and_stops(self): + with patch.dict(os.environ, {"WL_LOGGER_FLUSH_INTERVAL_SECONDS": "0.05"}): + lg = _lg() + self.addCleanup(lg.stop_background_flush) + self.assertIsNotNone(lg._flush_thread) + self.assertTrue(lg._flush_thread.is_alive()) + lg.stop_background_flush() + self.assertFalse(lg._flush_thread.is_alive()) + + def test_auto_detected_signals_are_classified_with_no_setup_call(self): + """The core ask: zero user-facing calls — just discovered and tagged.""" + lg = self._lg_no_autoflush() + with patch("weightslab.src.auto_loss_shape_signal_names", return_value=["train-loss-CE"]), \ + patch("weightslab.src.write_signal_shapes") as mock_write_shapes: + lg._autotag_loss_shapes() + self.assertTrue(mock_write_shapes.called) + args, kwargs = mock_write_shapes.call_args + self.assertEqual(args[0], "train-loss-CE") + self.assertIsNone(kwargs.get("tag_name")) # default '_shape' naming + self.assertIsNone(kwargs.get("classifier")) # default loss classifier + + def test_no_signals_means_no_classification_attempts(self): + lg = self._lg_no_autoflush() + with patch("weightslab.src.auto_loss_shape_signal_names", return_value=[]), \ + patch("weightslab.src.write_signal_shapes") as mock_write_shapes: + lg._autotag_loss_shapes() + self.assertFalse(mock_write_shapes.called) + + def test_set_loss_shape_override_customizes_tag_and_classifier(self): + lg = self._lg_no_autoflush() + classifier = MagicMock() + lg.set_loss_shape_override("train-loss-CE", tag_name="my_shape", classifier=classifier) + with patch("weightslab.src.auto_loss_shape_signal_names", return_value=["train-loss-CE"]), \ + patch("weightslab.src.write_signal_shapes") as mock_write_shapes: + lg._autotag_loss_shapes() + self.assertTrue(mock_write_shapes.called) + args, kwargs = mock_write_shapes.call_args + self.assertEqual(args[0], "train-loss-CE") + self.assertEqual(kwargs.get("tag_name"), "my_shape") + self.assertIs(kwargs.get("classifier"), classifier) + + def test_set_loss_shape_override_also_tracks_a_signal_outside_the_auto_detected_set(self): + """A manually-logged signal (not registered via flag="loss") can still + opt in by name — the override set is a superset, not a subset.""" + lg = self._lg_no_autoflush() + lg.set_loss_shape_override("custom_metric") + with patch("weightslab.src.auto_loss_shape_signal_names", return_value=[]), \ + patch("weightslab.src.write_signal_shapes") as mock_write_shapes: + lg._autotag_loss_shapes() + self.assertTrue(mock_write_shapes.called) + self.assertEqual(mock_write_shapes.call_args[0][0], "custom_metric") + + def test_disable_loss_shape_autotag_for_one_signal(self): + lg = self._lg_no_autoflush() + lg.disable_loss_shape_autotag("train-loss-CE") + with patch("weightslab.src.auto_loss_shape_signal_names", + return_value=["train-loss-CE", "val-loss-CE"]), \ + patch("weightslab.src.write_signal_shapes") as mock_write_shapes: + lg._autotag_loss_shapes() + called_signals = {c.args[0] for c in mock_write_shapes.call_args_list} + self.assertNotIn("train-loss-CE", called_signals) + self.assertIn("val-loss-CE", called_signals) + + def test_disable_loss_shape_autotag_for_all_signals(self): + lg = self._lg_no_autoflush() + lg.disable_loss_shape_autotag() + with patch("weightslab.src.auto_loss_shape_signal_names", + return_value=["train-loss-CE", "val-loss-CE"]), \ + patch("weightslab.src.write_signal_shapes") as mock_write_shapes: + lg._autotag_loss_shapes() + self.assertFalse(mock_write_shapes.called) + + def test_autotag_skips_a_signal_with_no_new_data_since_last_pass(self): + """The optimization under test: a signal with no new per-sample writes + since the last tag pass must not be reclassified again — this is what + keeps the background flush thread from unconditionally re-reading and + re-tagging every signal on a bare timer regardless of whether training + produced anything new.""" + lg = self._lg_no_autoflush() + _add(lg, "train-loss-CE", "s1", 1, 0.5) + with patch("weightslab.src.auto_loss_shape_signal_names", return_value=["train-loss-CE"]), \ + patch("weightslab.src.write_signal_shapes") as mock_write_shapes: + lg._autotag_loss_shapes() + self.assertEqual(mock_write_shapes.call_count, 1) + + # No new data staged for this signal — must be skipped this time. + lg._autotag_loss_shapes() + self.assertEqual(mock_write_shapes.call_count, 1) + + # New data arrives -> the signal's _qps_version moves -> re-tagged. + _add(lg, "train-loss-CE", "s1", 2, 0.4) + lg._autotag_loss_shapes() + self.assertEqual(mock_write_shapes.call_count, 2) + + def test_autotag_skip_cache_is_per_signal(self): + """One signal having no new data must not suppress another signal that + does — the version check is keyed per signal_name. Uses a second + sample at the SAME step (not a new global step) so only the touched + signal's own _qps_version bumps — a new global step drops the whole + query-cache/version table (see _stage_sample_row's "New step -> last + step's cache entries can't recur" comment), which is a separate, + coarser invalidation this test isn't about.""" + lg = self._lg_no_autoflush() + _add(lg, "train-loss-CE", "s1", 1, 0.5) + _add(lg, "val-loss-CE", "s1", 1, 0.7) + with patch("weightslab.src.auto_loss_shape_signal_names", + return_value=["train-loss-CE", "val-loss-CE"]), \ + patch("weightslab.src.write_signal_shapes") as mock_write_shapes: + lg._autotag_loss_shapes() + # Both signals are new — order isn't guaranteed (iterates a set). + self.assertEqual( + {c.args[0] for c in mock_write_shapes.call_args_list}, + {"train-loss-CE", "val-loss-CE"}, + ) + mock_write_shapes.reset_mock() + + # Only val-loss-CE gets a new sample this round (same step) — + # train-loss-CE must be skipped now. + _add(lg, "val-loss-CE", "s2", 1, 0.6) + lg._autotag_loss_shapes() + self.assertEqual( + [c.args[0] for c in mock_write_shapes.call_args_list], + ["val-loss-CE"], + ) + + +class TestAutoLossShapeSignalRegistration(unittest.TestCase): + """watch_or_edit(..., flag="loss") must register the signal name for + automatic loss-shape classification with no further call required.""" + + def test_flag_loss_registers_signal_for_auto_classification(self): + import torch.nn as nn + import weightslab as wl + from weightslab.src import _AUTO_LOSS_SHAPE_SIGNALS + + # _AUTO_LOSS_SHAPE_SIGNALS is process-global — clean up so this signal + # name doesn't leak into other tests' background flush threads (which + # would otherwise try, and safely fail, to classify a signal that + # doesn't exist in their own logger). + self.addCleanup(_AUTO_LOSS_SHAPE_SIGNALS.discard, "test_auto_loss_shape_signal") + + crit = nn.CrossEntropyLoss(reduction="none") + wl.watch_or_edit(crit, flag="loss", signal_name="test_auto_loss_shape_signal", log=True) + self.assertIn("test_auto_loss_shape_signal", wl.auto_loss_shape_signal_names()) + + def test_flag_metric_does_not_register_for_auto_classification(self): + """Only flag="loss" implies a decreasing per-sample loss trajectory — + an arbitrary watched metric/signal shouldn't be force-classified as one.""" + import torch + from torchmetrics.classification import Accuracy + import weightslab as wl + + wl.watch_or_edit(Accuracy(task="multiclass", num_classes=2), + flag="metric", signal_name="test_auto_metric_not_loss") + self.assertNotIn("test_auto_metric_not_loss", wl.auto_loss_shape_signal_names()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/backend/test_ui_docker_bridge.py b/tests/backend/test_ui_docker_bridge.py deleted file mode 100644 index c43f2bd5..00000000 --- a/tests/backend/test_ui_docker_bridge.py +++ /dev/null @@ -1,718 +0,0 @@ -import argparse -import contextlib -import io -import os -import sys -import unittest -from pathlib import Path -from unittest.mock import patch, MagicMock - -from weightslab.ui_docker_bridge import ( - _check_docker, - _clean_stale_docker_resources, - _compose_cmd, - _ensure_certificates, - _ensure_scripts_executable, - _generate_certs_with_fallback, - _get_example_dir, - _install_example_requirements, - _make_executable, - _remove_docker_image, - _strip_derived_deploy_env, - _DERIVED_DEPLOY_ENV_VARS, - _FRONTEND_IMAGE, - _STACK_CONTAINERS, - example_start, - main, - ui_launch, - ui_secure_environment, -) - - -class TestCheckDocker(unittest.TestCase): - @patch("weightslab.ui_docker_bridge.shutil.which", return_value=None) - def test_exits_when_docker_not_found(self, _mock_which): - with self.assertRaises(SystemExit) as ctx: - _check_docker() - self.assertEqual(ctx.exception.code, 1) - - @patch( - "weightslab.ui_docker_bridge.subprocess.run", - side_effect=__import__("subprocess").CalledProcessError(1, "docker info"), - ) - @patch("weightslab.ui_docker_bridge.shutil.which", return_value="/usr/bin/docker") - def test_exits_when_daemon_not_running(self, _mock_which, _mock_run): - with self.assertRaises(SystemExit) as ctx: - _check_docker() - self.assertEqual(ctx.exception.code, 1) - - @patch("weightslab.ui_docker_bridge.subprocess.run") - @patch("weightslab.ui_docker_bridge.shutil.which", return_value="/usr/bin/docker") - def test_passes_when_docker_available(self, _mock_which, mock_run): - _check_docker() - mock_run.assert_called_once() - - -class TestComposeCmd(unittest.TestCase): - @patch("weightslab.ui_docker_bridge._compose_base_cmd", return_value=["docker", "compose"]) - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_runs_docker_compose_with_env(self, mock_run, _mock_base): - mock_run.return_value = MagicMock(stdout="", returncode=0) - _compose_cmd("/path/to/compose.yml", "/path/to/envoy.yaml", ["up", "-d"]) - mock_run.assert_called_once() - args, kwargs = mock_run.call_args - self.assertEqual( - args[0], - ["docker", "compose", "-f", "/path/to/compose.yml", "up", "-d"], - ) - self.assertEqual(kwargs["env"]["WS_ENVOY_CONFIG"], "/path/to/envoy.yaml") - self.assertTrue(kwargs["stdout"]) - self.assertTrue(kwargs["text"]) - - @patch("weightslab.ui_docker_bridge._compose_base_cmd", return_value=["docker-compose"]) - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_v1_translates_up_pull_into_pull_then_up(self, mock_run, _mock_base): - """Compose v1 has no `up --pull`; it must `pull` first, then `up` without the flag.""" - mock_run.return_value = MagicMock(stdout="", returncode=0) - _compose_cmd("/c.yml", "/e.yaml", ["up", "-d", "--pull", "always"]) - cmds = [c.args[0] for c in mock_run.call_args_list] - # A separate `docker-compose -f /c.yml pull` ran first. - self.assertIn(["docker-compose", "-f", "/c.yml", "pull"], cmds) - # The final `up` carries no --pull flag (v1 would reject it). - up_cmd = cmds[-1] - self.assertEqual(up_cmd, ["docker-compose", "-f", "/c.yml", "up", "-d"]) - self.assertNotIn("--pull", up_cmd) - - @patch("weightslab.ui_docker_bridge._compose_base_cmd", return_value=None) - def test_exits_when_no_compose_cli(self, _mock_base): - with self.assertRaises(SystemExit) as ctx: - _compose_cmd("/c.yml", "/e.yaml", ["up", "-d"]) - self.assertEqual(ctx.exception.code, 1) - - -class TestComposeDetection(unittest.TestCase): - """Prefer Compose v2 (`docker compose`), fall back to v1 (`docker-compose`).""" - - @staticmethod - def _fake_run(ok_keys): - """Return a subprocess.run stub: rc 0 when ' '.join(cmd[:2]) is in ok_keys.""" - def run(cmd, *a, **k): - return MagicMock(returncode=0 if " ".join(cmd[:2]) in ok_keys else 1) - return run - - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_prefers_v2(self, mock_run): - from weightslab.ui_docker_bridge import _detect_compose_cmd - mock_run.side_effect = self._fake_run({"docker compose"}) - self.assertEqual(_detect_compose_cmd(), ["docker", "compose"]) - - @patch("weightslab.ui_docker_bridge.shutil.which", return_value="/usr/bin/docker-compose") - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_falls_back_to_v1_when_v2_absent(self, mock_run, _which): - from weightslab.ui_docker_bridge import _detect_compose_cmd - # v2 probe fails (rc 1); v1 probe (`docker-compose version`) succeeds. - mock_run.side_effect = self._fake_run({"docker-compose version"}) - self.assertEqual(_detect_compose_cmd(), ["docker-compose"]) - - @patch("weightslab.ui_docker_bridge.shutil.which", return_value=None) - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_returns_none_when_neither_available(self, mock_run, _which): - from weightslab.ui_docker_bridge import _detect_compose_cmd - mock_run.side_effect = self._fake_run(set()) - self.assertIsNone(_detect_compose_cmd()) - - -class TestScriptsExecutable(unittest.TestCase): - """Bundled .sh scripts are made executable so users skip the manual `chmod +x`.""" - - @unittest.skipIf(sys.platform == "win32", "execute bit is POSIX-only") - def test_make_executable_adds_exec_bits(self): - import stat as _stat - import tempfile - with tempfile.NamedTemporaryFile(suffix=".sh", delete=False) as f: - path = f.name - try: - os.chmod(path, 0o644) - _make_executable(path) - mode = os.stat(path).st_mode - self.assertTrue(mode & _stat.S_IXUSR) - self.assertTrue(mode & _stat.S_IXGRP) - self.assertTrue(mode & _stat.S_IXOTH) - finally: - os.unlink(path) - - def test_make_executable_is_noop_on_windows(self): - with patch("weightslab.ui_docker_bridge._is_windows", return_value=True): - with patch("weightslab.ui_docker_bridge.os.chmod") as mock_chmod: - _make_executable("/whatever/path.sh") - mock_chmod.assert_not_called() - - def test_make_executable_swallows_oserror(self): - # A non-chmod-able path (e.g. root-owned system install) must not raise. - with patch("weightslab.ui_docker_bridge.os.stat", side_effect=OSError("denied")): - _make_executable("/root/owned.sh") # should not raise - - @unittest.skipIf(sys.platform == "win32", "execute bit is POSIX-only") - def test_ensure_scripts_executable_marks_bundled_scripts(self): - import stat as _stat - from weightslab.ui_docker_bridge import _get_bootstrap_script - _ensure_scripts_executable() - bootstrap = _get_bootstrap_script() - self.assertTrue(bootstrap.exists(), bootstrap) - self.assertTrue(os.stat(bootstrap).st_mode & _stat.S_IXUSR) - - def test_ensure_scripts_executable_noop_on_windows(self): - with patch("weightslab.ui_docker_bridge._is_windows", return_value=True): - with patch("weightslab.ui_docker_bridge._make_executable") as mock_mk: - _ensure_scripts_executable() - mock_mk.assert_not_called() - - -class TestUiLaunch(unittest.TestCase): - """ui_launch: unsecured by default (no cert gen); --certs opts into TLS.""" - - @patch("weightslab.ui_docker_bridge.CertAuthManager") - @patch("weightslab.ui_docker_bridge._get_bootstrap_script", return_value="/does/not/exist.sh") - @patch("weightslab.ui_docker_bridge._run_shell_script", return_value=0) - @patch("weightslab.ui_docker_bridge._ensure_certificates", return_value=True) - @patch("weightslab.ui_docker_bridge._clean_stale_docker_resources") - @patch("weightslab.ui_docker_bridge._compose_cmd") - @patch("weightslab.ui_docker_bridge._check_docker") - @patch("weightslab.ui_docker_bridge._get_envoy_config", return_value="/fake/envoy.yaml") - @patch("weightslab.ui_docker_bridge._get_compose_file", return_value="/fake/docker-compose.yml") - def test_launch_default_no_cert_gen_cleans_and_launches_unsecured( - self, _gc, _ge, mock_check, mock_compose, mock_clean, mock_ensure, - _mock_shell, _gb, mock_mgr, - ): - mgr = MagicMock() - mgr.has_valid_certs.return_value = False # no certs on disk -> unsecured - mock_mgr.from_env_or_default.return_value = mgr - with patch.dict(os.environ, {}, clear=False): - os.environ.pop("VITE_PORT", None) - with self.assertLogs("weightslab.ui_docker_bridge", level="INFO") as log_context: - ui_launch(argparse.Namespace()) - mock_check.assert_called_once() - mock_ensure.assert_not_called() # certs NOT generated by default - mock_clean.assert_called_once() # stale cleanup ran - mock_compose.assert_called_once_with( - "/fake/docker-compose.yml", - "/fake/envoy.yaml", - ["up", "-d", "--pull", "always"], - ) - self.assertTrue(any("http://localhost:5173" in msg for msg in log_context.output)) - self.assertFalse(any("https://localhost:5173" in msg for msg in log_context.output)) - - @patch("weightslab.ui_docker_bridge.CertAuthManager") - @patch("weightslab.ui_docker_bridge._get_bootstrap_script", return_value="/does/not/exist.sh") - @patch("weightslab.ui_docker_bridge._run_shell_script", return_value=0) - @patch("weightslab.ui_docker_bridge._ensure_certificates", return_value=True) - @patch("weightslab.ui_docker_bridge._clean_stale_docker_resources") - @patch("weightslab.ui_docker_bridge._compose_cmd") - @patch("weightslab.ui_docker_bridge._check_docker") - @patch("weightslab.ui_docker_bridge._get_envoy_config", return_value="/fake/envoy.yaml") - @patch("weightslab.ui_docker_bridge._get_compose_file", return_value="/fake/docker-compose.yml") - def test_launch_respects_custom_port( - self, _gc, _ge, _mock_check, _mock_compose, _mock_clean, _mock_ensure, - _mock_shell, _gb, mock_mgr, - ): - mgr = MagicMock(certs_dir="/fake/certs") - mgr.has_valid_certs.return_value = True - mock_mgr.from_env_or_default.return_value = mgr - with patch.dict(os.environ, {"VITE_PORT": "3000"}): - with self.assertLogs("weightslab.ui_docker_bridge", level="INFO") as log_context: - ui_launch(argparse.Namespace(certs=True)) - self.assertTrue(any("https://localhost:3000" in msg for msg in log_context.output)) - - @patch("weightslab.ui_docker_bridge.CertAuthManager") - @patch("weightslab.ui_docker_bridge._get_bootstrap_script", return_value="/does/not/exist.sh") - @patch("weightslab.ui_docker_bridge._run_shell_script", return_value=0) - @patch("weightslab.ui_docker_bridge._ensure_certificates", return_value=True) - @patch("weightslab.ui_docker_bridge._clean_stale_docker_resources") - @patch("weightslab.ui_docker_bridge._compose_cmd") - @patch("weightslab.ui_docker_bridge._check_docker") - @patch("weightslab.ui_docker_bridge._get_envoy_config", return_value="/fake/envoy.yaml") - @patch("weightslab.ui_docker_bridge._get_compose_file", return_value="/fake/docker-compose.yml") - def test_launch_certs_flag_generates_and_runs_secured( - self, _gc, _ge, _mock_check, _mock_compose, _mock_clean, mock_ensure, - _mock_shell, _gb, mock_mgr, - ): - mgr = MagicMock(certs_dir="/fake/certs") - mgr.has_valid_certs.return_value = True # certs present after generation - mock_mgr.from_env_or_default.return_value = mgr - with patch.dict(os.environ, {}, clear=False): - os.environ.pop("VITE_PORT", None) - with self.assertLogs("weightslab.ui_docker_bridge", level="INFO") as log_context: - ui_launch(argparse.Namespace(certs=True)) - mock_ensure.assert_called_once() # --certs generates certs - self.assertTrue(any("https://localhost:5173" in msg for msg in log_context.output)) - - -class TestEnsureCertificates(unittest.TestCase): - """_ensure_certificates only generates files; it never exports TLS/auth env.""" - - @patch("weightslab.ui_docker_bridge._install_ca_trust") - @patch("weightslab.ui_docker_bridge._generate_certs_with_fallback") - def test_uses_existing_certs_without_generating(self, mock_gen, _mock_trust): - manager = MagicMock() - # Gate is has_any_credentials(); existing creds short-circuit generation. - manager.has_any_credentials.return_value = True - manager.has_valid_certs.return_value = True - result = _ensure_certificates(manager, force_certs=False) - self.assertTrue(result) - mock_gen.assert_not_called() - manager.get_or_create_auth_token.assert_called_once() - # Derived TLS env must NOT be set here — the deploy pipeline derives it. - manager.setup_tls_environment.assert_not_called() - - @patch("weightslab.ui_docker_bridge._install_ca_trust") - @patch("weightslab.ui_docker_bridge._generate_certs_with_fallback", return_value=0) - def test_generates_when_missing_and_forwards_certs_dir(self, mock_gen, _mock_trust): - manager = MagicMock() - # No credentials at the gate -> generate; certs valid afterwards. - manager.has_any_credentials.return_value = False - manager.has_valid_certs.return_value = True - result = _ensure_certificates(manager, force_certs=False) - self.assertTrue(result) - mock_gen.assert_called_once_with(force_certs=False, certs_dir=manager.certs_dir) - manager.setup_tls_environment.assert_not_called() - - @patch("weightslab.ui_docker_bridge._install_ca_trust") - @patch("weightslab.ui_docker_bridge._generate_certs_with_fallback", return_value=0) - def test_force_regenerates_even_when_present(self, mock_gen, _mock_trust): - manager = MagicMock() - manager.has_any_credentials.return_value = True - manager.has_valid_certs.return_value = True - _ensure_certificates(manager, force_certs=True) - mock_gen.assert_called_once_with(force_certs=True, certs_dir=manager.certs_dir) - - @patch("weightslab.ui_docker_bridge._install_ca_trust") - @patch("weightslab.ui_docker_bridge._generate_certs_with_fallback", return_value=1) - def test_returns_false_on_generation_failure(self, mock_gen, _mock_trust): - manager = MagicMock() - manager.has_any_credentials.return_value = False - manager.has_valid_certs.return_value = False - result = _ensure_certificates(manager, force_certs=False) - self.assertFalse(result) - - -class TestRemoveDockerImage(unittest.TestCase): - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_removes_when_present(self, mock_run): - mock_run.side_effect = [MagicMock(stdout="abc123\nabc123\ndef456\n"), MagicMock()] - _remove_docker_image(_FRONTEND_IMAGE) - self.assertEqual(mock_run.call_count, 2) - rmi_call = mock_run.call_args_list[1].args[0] - self.assertEqual(rmi_call[:3], ["docker", "rmi", "-f"]) - self.assertIn("abc123", rmi_call) - self.assertIn("def456", rmi_call) - - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_noop_when_absent(self, mock_run): - mock_run.return_value = MagicMock(stdout="") - _remove_docker_image(_FRONTEND_IMAGE) - mock_run.assert_called_once() # only the 'docker images -q' query, no rmi - - -class TestCleanStaleDockerResources(unittest.TestCase): - @patch("weightslab.ui_docker_bridge._remove_docker_image") - @patch("weightslab.ui_docker_bridge.subprocess.run") - @patch("weightslab.ui_docker_bridge._compose_cmd") - @patch("weightslab.ui_docker_bridge._get_envoy_config", return_value="/fake/envoy.yaml") - @patch("weightslab.ui_docker_bridge._get_compose_file", return_value="/fake/docker-compose.yml") - def test_clean_tears_down_and_removes_image(self, _gc, _ge, mock_compose, mock_run, mock_rmimg): - _clean_stale_docker_resources() - # 1. compose down --remove-orphans --volumes - mock_compose.assert_called_once_with( - "/fake/docker-compose.yml", - "/fake/envoy.yaml", - ["down", "--remove-orphans", "--volumes"], - ) - # 2. docker rm -f for each known stack container - removed = [c.args[0] for c in mock_run.call_args_list] - for container in _STACK_CONTAINERS: - self.assertTrue( - any(call[:3] == ["docker", "rm", "-f"] and container in call for call in removed), - f"expected 'docker rm -f {container}'", - ) - # 3. cached frontend image removed - mock_rmimg.assert_called_once_with(_FRONTEND_IMAGE) - - @patch("weightslab.ui_docker_bridge._remove_docker_image") - @patch("weightslab.ui_docker_bridge.subprocess.run") - @patch("weightslab.ui_docker_bridge._compose_cmd", - side_effect=__import__("subprocess").CalledProcessError(1, "down")) - @patch("weightslab.ui_docker_bridge._get_envoy_config", return_value="/fake/envoy.yaml") - @patch("weightslab.ui_docker_bridge._get_compose_file", return_value="/fake/docker-compose.yml") - def test_clean_tolerates_compose_down_failure(self, _gc, _ge, _mc, _mr, mock_rmimg): - # Should not raise even when 'compose down' returns non-zero (nothing to remove). - _clean_stale_docker_resources() - mock_rmimg.assert_called_once_with(_FRONTEND_IMAGE) - - -class TestUiSecureEnvironment(unittest.TestCase): - @patch("weightslab.ui_docker_bridge.CertAuthManager") - @patch("weightslab.ui_docker_bridge._generate_certs_with_fallback", return_value=0) - def test_ui_secure_environment_success(self, mock_gen_certs, mock_cert_manager): - """`weightslab se`: generate certs + token, export WEIGHTSLAB_CERTS_DIR.""" - mock_manager_instance = MagicMock() # certs_dir is a MagicMock (supports .mkdir) - mock_manager_instance.get_or_create_auth_token.return_value = "fake_token" - mock_cert_manager.return_value = mock_manager_instance - - args = argparse.Namespace(force_certs=False) - - with self.assertLogs("weightslab.ui_docker_bridge", level="INFO") as log_context: - ui_secure_environment(args) - - self.assertTrue(any("Certificates generated successfully" in msg for msg in log_context.output)) - self.assertTrue(any("gRPC auth token created" in msg for msg in log_context.output)) - # Generation is pointed at the chosen certs dir (single source of truth). - mock_gen_certs.assert_called_once_with(force_certs=False, certs_dir=mock_manager_instance.certs_dir) - mock_manager_instance.certs_dir.mkdir.assert_called_once() - # se exports WEIGHTSLAB_CERTS_DIR for the process. - self.assertTrue(any("WEIGHTSLAB_CERTS_DIR exported" in msg for msg in log_context.output)) - - @patch("weightslab.ui_docker_bridge._generate_certs_with_fallback", return_value=0) - def test_ui_secure_environment_force_certs(self, mock_gen_certs): - """`weightslab se --force-certs` forwards force_certs to generation.""" - args = argparse.Namespace(force_certs=True) - with patch.dict(os.environ, {}, clear=False): - ui_secure_environment(args) - self.assertTrue(mock_gen_certs.call_args.kwargs["force_certs"]) - - @patch("weightslab.ui_docker_bridge._generate_certs_with_fallback", return_value=1) - def test_ui_secure_environment_cert_failure(self, mock_gen_certs): - """Certificate generation failure exits non-zero.""" - args = argparse.Namespace(force_certs=False) - with self.assertRaises(SystemExit) as ctx: - ui_secure_environment(args) - self.assertEqual(ctx.exception.code, 1) - - -class TestMainCLI(unittest.TestCase): - """The CLI exposes exactly: se, ui launch, start example, help.""" - - @patch("weightslab.ui_docker_bridge.ui_secure_environment") - def test_main_dispatches_se(self, mock_se): - with patch("sys.argv", ["weightslab", "se"]): - main() - mock_se.assert_called_once() - - @patch("weightslab.ui_docker_bridge.ui_launch") - def test_main_dispatches_ui_launch(self, mock_launch): - with patch("sys.argv", ["weightslab", "ui", "launch"]): - main() - mock_launch.assert_called_once() - - @patch("weightslab.ui_docker_bridge.example_start") - def test_main_dispatches_start_example(self, mock_example): - with patch("sys.argv", ["weightslab", "start", "example"]): - main() - mock_example.assert_called_once() - - def test_main_ui_without_action_does_not_crash(self): - with patch("sys.argv", ["weightslab", "ui"]): - main() # should print ui help, not raise - - def test_main_start_without_target_does_not_crash(self): - with patch("sys.argv", ["weightslab", "start"]): - main() # should print start help, not raise - - def test_main_help_does_not_crash(self): - with patch("sys.argv", ["weightslab", "help"]): - main() # should not raise - - def test_main_no_args_does_not_crash(self): - with patch("sys.argv", ["weightslab"]): - main() # should not raise - - -class TestUserOnboardingFlow(unittest.TestCase): - """Integration-like test of the full onboarding flow, fully hermetic.""" - - @patch("weightslab.ui_docker_bridge.subprocess.run") - @patch("weightslab.ui_docker_bridge._compose_cmd") - @patch("weightslab.ui_docker_bridge._check_docker") - @patch("weightslab.ui_docker_bridge._run_shell_script", return_value=0) - @patch("weightslab.ui_docker_bridge._clean_stale_docker_resources") - @patch("weightslab.ui_docker_bridge._ensure_certificates", return_value=True) - @patch("weightslab.ui_docker_bridge._generate_certs_with_fallback", return_value=0) - @patch("weightslab.ui_docker_bridge.CertAuthManager") - def test_complete_onboarding_workflow( - self, mock_cert_manager, mock_gen, mock_ensure, mock_clean, - mock_shell, mock_check, mock_compose, mock_subproc, - ): - """se -> ui launch -> start example, with no real Docker/daemon/subprocess.""" - mgr = MagicMock() - mock_cert_manager.return_value = mgr - mock_cert_manager.from_env_or_default.return_value = mgr - mock_subproc.return_value = MagicMock(returncode=0) - - with patch.dict(os.environ, {}, clear=False): - try: - ui_secure_environment(argparse.Namespace(force_certs=False)) - ui_launch(argparse.Namespace()) - example_start(argparse.Namespace()) - except Exception as e: - self.fail(f"Onboarding workflow failed: {e}") - - @patch("sys.argv", ["weightslab", "se", "--force-certs"]) - @patch("weightslab.ui_docker_bridge.ui_secure_environment") - def test_cli_se_force_certs(self, mock_se): - main() - mock_se.assert_called_once() - self.assertTrue(mock_se.call_args.args[0].force_certs) - - -class TestBackendConnectionDetection(unittest.TestCase): - """Test backend connection detection utility.""" - - @patch("socket.socket") - def test_backend_connection_success(self, mock_socket_class): - from weightslab.ui_docker_bridge import _test_backend_connection - mock_socket = MagicMock() - mock_socket.connect_ex.return_value = 0 - mock_socket_class.return_value = mock_socket - self.assertTrue(_test_backend_connection()) - - @patch("socket.socket") - def test_backend_connection_failure(self, mock_socket_class): - from weightslab.ui_docker_bridge import _test_backend_connection - mock_socket = MagicMock() - mock_socket.connect_ex.return_value = 1 - mock_socket_class.return_value = mock_socket - self.assertFalse(_test_backend_connection()) - - @patch("socket.socket") - def test_backend_connection_timeout(self, mock_socket_class): - from weightslab.ui_docker_bridge import _test_backend_connection - mock_socket = MagicMock() - mock_socket.connect_ex.side_effect = Exception("Connection timeout") - mock_socket_class.return_value = mock_socket - self.assertFalse(_test_backend_connection()) - - -class TestPathConversion(unittest.TestCase): - """Test Windows path to Git Bash conversion.""" - - def test_windows_path_conversion(self): - from weightslab.ui_docker_bridge import _convert_to_git_bash_path - win_path = r"C:\Users\testuser\.weightslab-certs" - bash_path = _convert_to_git_bash_path(win_path) - self.assertEqual(bash_path, "/mnt/c/Users/testuser/.weightslab-certs") - - def test_unix_path_passthrough(self): - from weightslab.ui_docker_bridge import _convert_to_git_bash_path - unix_path = "/home/testuser/.weightslab-certs" - self.assertEqual(_convert_to_git_bash_path(unix_path), unix_path) - - -class TestSingleSourceOfTruth(unittest.TestCase): - """WEIGHTSLAB_CERTS_DIR is the only input; everything else derives from it.""" - - def test_strip_derived_deploy_env_removes_all(self): - sentinel = {k: "stale" for k in _DERIVED_DEPLOY_ENV_VARS} - with patch.dict(os.environ, sentinel, clear=False): - _strip_derived_deploy_env() - for key in _DERIVED_DEPLOY_ENV_VARS: - self.assertNotIn(key, os.environ, f"{key} should have been stripped") - - @patch("weightslab.ui_docker_bridge._run_shell_script", return_value=0) - def test_generate_certs_forwards_certs_dir(self, mock_shell): - rc = _generate_certs_with_fallback(force_certs=False, certs_dir="/custom/certs") - self.assertEqual(rc, 0) - env_vars = mock_shell.call_args.args[2] - self.assertIsInstance(env_vars, dict) - self.assertIn("WEIGHTSLAB_CERTS_DIR", env_vars) - self.assertIn("custom/certs", env_vars["WEIGHTSLAB_CERTS_DIR"].replace("\\", "/")) - - @patch("weightslab.ui_docker_bridge._run_shell_script", return_value=0) - def test_generate_certs_without_dir_passes_no_env(self, mock_shell): - _generate_certs_with_fallback(force_certs=False) - self.assertIsNone(mock_shell.call_args.args[2]) - - @patch("weightslab.ui_docker_bridge.CertAuthManager") - @patch("weightslab.ui_docker_bridge._get_bootstrap_script", return_value="/does/not/exist.sh") - @patch("weightslab.ui_docker_bridge._ensure_certificates", return_value=True) - @patch("weightslab.ui_docker_bridge._clean_stale_docker_resources") - @patch("weightslab.ui_docker_bridge._compose_cmd") - @patch("weightslab.ui_docker_bridge._check_docker") - @patch("weightslab.ui_docker_bridge._get_envoy_config", return_value="/fake/envoy.yaml") - @patch("weightslab.ui_docker_bridge._get_compose_file", return_value="/fake/docker-compose.yml") - def test_ui_launch_strips_derived_env( - self, _gc, _ge, _mock_check, _mock_compose, _mock_clean, _mock_ensure, _gb, mock_mgr, - ): - mock_mgr.from_env_or_default.return_value = MagicMock() - # Simulate a stale derived env var leaking in (e.g. from import-time check). - with patch.dict(os.environ, {"ENVOY_DOWNSTREAM_TLS": "on", "VITE_SERVER_PROTOCOL": "https"}, clear=False): - ui_launch(argparse.Namespace()) - self.assertNotIn("ENVOY_DOWNSTREAM_TLS", os.environ) - self.assertNotIn("VITE_SERVER_PROTOCOL", os.environ) - - @patch("weightslab.ui_docker_bridge.CertAuthManager") - @patch("weightslab.ui_docker_bridge._get_bootstrap_script", return_value="/does/not/exist.sh") - @patch("weightslab.ui_docker_bridge._ensure_certificates", return_value=True) - @patch("weightslab.ui_docker_bridge._clean_stale_docker_resources") - @patch("weightslab.ui_docker_bridge._compose_cmd") - @patch("weightslab.ui_docker_bridge._check_docker") - @patch("weightslab.ui_docker_bridge._get_envoy_config", return_value="/fake/envoy.yaml") - @patch("weightslab.ui_docker_bridge._get_compose_file", return_value="/fake/docker-compose.yml") - def test_ui_launch_url_https_only_when_certs_present( - self, _gc, _ge, _mock_check, _mock_compose, _mock_clean, _mock_ensure, _gb, mock_mgr, - ): - # Certs absent on disk -> URL must be http (default unsecured launch). - mgr = MagicMock() - mgr.has_valid_certs.return_value = False - mock_mgr.from_env_or_default.return_value = mgr - with patch.dict(os.environ, {}, clear=False): - os.environ.pop("VITE_PORT", None) - with self.assertLogs("weightslab.ui_docker_bridge", level="INFO") as log_context: - ui_launch(argparse.Namespace()) - self.assertTrue(any("http://localhost:5173" in m for m in log_context.output)) - self.assertFalse(any("https://localhost:5173" in m for m in log_context.output)) - - -class TestExampleStart(unittest.TestCase): - """`weightslab start example` runs the bundled classification example.""" - - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_example_start_runs_classification(self, mock_run): - mock_run.return_value = MagicMock(returncode=0) - with self.assertLogs("weightslab.ui_docker_bridge", level="INFO") as log_context: - example_start(argparse.Namespace()) - mock_run.assert_called_once() - cmd = mock_run.call_args.args[0] - self.assertEqual(cmd[0], sys.executable) - main_py = cmd[1].replace("\\", "/") - self.assertTrue(main_py.endswith("examples/PyTorch/ws-classification/main.py"), main_py) - cwd = mock_run.call_args.kwargs["cwd"].replace("\\", "/") - self.assertTrue(cwd.endswith("examples/PyTorch/ws-classification"), cwd) - self.assertTrue(any("classification (cls) example" in m for m in log_context.output)) - - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_example_start_propagates_nonzero_exit(self, mock_run): - mock_run.return_value = MagicMock(returncode=3) - with self.assertRaises(SystemExit) as ctx: - example_start(argparse.Namespace()) - self.assertEqual(ctx.exception.code, 3) - - @patch("weightslab.ui_docker_bridge._get_example_dir", return_value=Path("/does/not/exist")) - def test_example_start_errors_when_missing(self, _mock_dir): - with self.assertRaises(SystemExit) as ctx: - example_start(argparse.Namespace()) - self.assertEqual(ctx.exception.code, 1) - - def test_example_dir_points_at_bundled_example(self): - # The bundled classification example must actually ship with the package. - self.assertTrue((_get_example_dir("ws-classification") / "main.py").exists()) - - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_example_start_seg_runs_segmentation(self, mock_run): - mock_run.return_value = MagicMock(returncode=0) - with self.assertLogs("weightslab.ui_docker_bridge", level="INFO") as log_context: - example_start(argparse.Namespace(example_kind="seg")) - main_py = mock_run.call_args.args[0][1].replace("\\", "/") - self.assertTrue(main_py.endswith("examples/PyTorch/ws-segmentation/main.py"), main_py) - self.assertTrue(any("segmentation (seg) example" in m for m in log_context.output)) - - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_example_start_defaults_to_cls_when_flag_absent(self, mock_run): - # A Namespace without example_kind (e.g. older call sites) still defaults to cls. - mock_run.return_value = MagicMock(returncode=0) - example_start(argparse.Namespace()) - main_py = mock_run.call_args.args[0][1].replace("\\", "/") - self.assertTrue(main_py.endswith("examples/PyTorch/ws-classification/main.py"), main_py) - - -class TestInstallExampleRequirements(unittest.TestCase): - """Requirements install is non-interactive and only runs when a file is present.""" - - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_installs_requirements_non_interactively_when_present(self, mock_run): - import tempfile - mock_run.return_value = MagicMock(returncode=0) - with tempfile.TemporaryDirectory() as tmp: - req = Path(tmp) / "requirements.txt" - req.write_text("numpy\n") - _install_example_requirements(Path(tmp)) - mock_run.assert_called_once() - cmd = mock_run.call_args.args[0] - self.assertEqual(cmd[:5], [sys.executable, "-m", "pip", "install", "-r"]) - self.assertIn("--no-input", cmd) # never prompts - self.assertTrue(mock_run.call_args.kwargs.get("check")) - - @patch("weightslab.ui_docker_bridge.subprocess.run") - def test_skips_when_no_requirements_file(self, mock_run): - import tempfile - with tempfile.TemporaryDirectory() as tmp: - _install_example_requirements(Path(tmp)) - mock_run.assert_not_called() - - -class TestBannerAndHelp(unittest.TestCase): - """`weightslab`, `weightslab help`, `-h`, `--help` show banner + the command set.""" - - @staticmethod - def _capture_main(argv): - buf = io.StringIO() - with patch("sys.argv", argv): - with contextlib.redirect_stdout(buf): - try: - main() - except SystemExit: - pass # argparse -h/--help exits 0 - return buf.getvalue() - - def test_dash_h_shows_banner_and_command_reference(self): - out = self._capture_main(["weightslab", "-h"]) - self.assertIn("WeightsLab", out) # tagline from description - self.assertIn("ui launch", out) - self.assertIn("--certs", out) - self.assertIn("start example", out) - - def test_long_help_flag_shows_command_reference(self): - out = self._capture_main(["weightslab", "--help"]) - self.assertIn("ui launch", out) - self.assertIn("--force-certs", out) - - def test_help_subcommand_shows_command_reference(self): - out = self._capture_main(["weightslab", "help"]) - self.assertIn("se", out) - self.assertIn("ui launch", out) - self.assertIn("start example", out) - - def test_no_args_shows_command_reference(self): - out = self._capture_main(["weightslab"]) - self.assertIn("ui launch", out) - - def test_help_does_not_mention_removed_commands(self): - out = self._capture_main(["weightslab", "help"]) - for removed in ("ui stop", "ui drop", "ui docker", "--no-auth", "--no-clean"): - self.assertNotIn(removed, out, f"help should not mention removed '{removed}'") - - -class TestLaunchCliFlags(unittest.TestCase): - """`ui launch` accepts only --certs (TLS is opt-in; default unsecured).""" - - @patch("weightslab.ui_docker_bridge.ui_launch") - def test_launch_certs_flag_parsed(self, mock_launch): - with patch("sys.argv", ["weightslab", "ui", "launch", "--certs"]): - main() - mock_launch.assert_called_once() - self.assertTrue(mock_launch.call_args.args[0].certs) - - @patch("weightslab.ui_docker_bridge.ui_launch") - def test_launch_default_certs_false(self, mock_launch): - with patch("sys.argv", ["weightslab", "ui", "launch"]): - main() - self.assertFalse(mock_launch.call_args.args[0].certs) - - def test_launch_removed_flags_are_rejected(self): - for flag in ("--no-auth", "--force-certs", "--no-clean", "--dev"): - with patch("sys.argv", ["weightslab", "ui", "launch", flag]): - with self.assertRaises(SystemExit) as ctx: - main() - self.assertNotEqual(ctx.exception.code, 0, flag) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/backend/test_write_dataframe.py b/tests/backend/test_write_dataframe.py index 163eec03..5582623c 100644 --- a/tests/backend/test_write_dataframe.py +++ b/tests/backend/test_write_dataframe.py @@ -48,6 +48,19 @@ def _call(path, manager, **kwargs): return write_dataframe(path, **kwargs) +def _records(path): + """Read a JSON export back into a list of row records. + + write_dataframe defaults to ``orient="columns"`` (``{column: {row: value}}``) + — compact (~6x smaller than records for wide sparse tables) and round-trips + with ``pd.read_json``'s default orient. These tests assert row-level + behavior, so normalize the columnar layout back to records here. Empty + exports (``{}`` / ``{"index": {}}``) collapse to ``[]``. + """ + raw = json.loads(open(path, encoding="utf-8").read()) + return pd.DataFrame(raw).to_dict(orient="records") + + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -89,20 +102,22 @@ def test_flush_failure_does_not_abort(self, tmp_json): # --------------------------------------------------------------------------- class TestWriteDataframeJsonStructure: - def test_json_is_list_of_records(self, mgr, tmp_json): + def test_json_is_columnar_dict(self, mgr, tmp_json): _call(tmp_json, mgr) - data = json.loads(open(tmp_json).read()) - assert isinstance(data, list) + raw = json.loads(open(tmp_json).read()) + # Default orient="columns": {column: {row_index: value}} + assert isinstance(raw, dict) + assert isinstance(raw["sample_id"], dict) def test_json_includes_index_as_columns(self, mgr, tmp_json): _call(tmp_json, mgr) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert "sample_id" in data[0] assert "annotation_id" in data[0] def test_json_all_rows_present_by_default(self, mgr, tmp_json): _call(tmp_json, mgr) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert len(data) == 4 def test_returns_written_path(self, mgr, tmp_json): @@ -207,30 +222,30 @@ def test_returns_path_when_no_manager(self, tmp_json): class TestWriteDataframeColumnFilters: def test_columns_all_keeps_everything(self, mgr, tmp_json): _call(tmp_json, mgr, columns="all") - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert "signals_loss" in data[0] or any("signals_loss" in r for r in data) def test_columns_tags_only(self, mgr, tmp_json): _call(tmp_json, mgr, columns="tags") - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) non_index = [k for k in data[0] if k not in ("sample_id", "annotation_id")] assert all(k.startswith("tag:") or k.startswith("TAG:") for k in non_index) def test_columns_signals_only(self, mgr, tmp_json): _call(tmp_json, mgr, columns="signals") - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) non_index = [k for k in data[0] if k not in ("sample_id", "annotation_id")] assert all(str(k).lower().startswith("signals") for k in non_index) def test_columns_discarded_only(self, mgr, tmp_json): _call(tmp_json, mgr, columns="discarded") - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) non_index = [k for k in data[0] if k not in ("sample_id", "annotation_id")] assert non_index == ["discarded"] def test_columns_list_of_groups(self, mgr, tmp_json): _call(tmp_json, mgr, columns=["tags", "discarded"]) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) non_index = set(k for r in data for k in r if k not in ("sample_id", "annotation_id")) assert "discarded" in non_index assert any(k.startswith("tag:") for k in non_index) @@ -238,20 +253,20 @@ def test_columns_list_of_groups(self, mgr, tmp_json): def test_columns_exact_name(self, mgr, tmp_json): _call(tmp_json, mgr, columns=["signals_loss"]) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) non_index = [k for k in data[0] if k not in ("sample_id", "annotation_id")] assert non_index == ["signals_loss"] def test_columns_nonexistent_name_yields_empty_cols(self, mgr, tmp_json): _call(tmp_json, mgr, columns=["nonexistent_col"]) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) # Index columns always present; no extra columns for row in data: assert set(row.keys()) <= {"sample_id", "annotation_id"} def test_columns_none_keeps_all(self, mgr, tmp_json): _call(tmp_json, mgr, columns=None) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert "signals_loss" in data[0] or any("signals_loss" in r for r in data) @@ -262,38 +277,38 @@ def test_columns_none_keeps_all(self, mgr, tmp_json): class TestWriteDataframeIndexFilters: def test_sample_id_single(self, mgr, tmp_json): _call(tmp_json, mgr, sample_id="s1") - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert all(r["sample_id"] == "s1" for r in data) assert len(data) == 2 # s1 has annotation_ids 0 and 1 def test_sample_id_list(self, mgr, tmp_json): _call(tmp_json, mgr, sample_id=["s1", "s2"]) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) sids = {r["sample_id"] for r in data} assert sids == {"s1", "s2"} assert len(data) == 4 def test_instance_id_zero_keeps_sample_rows(self, mgr, tmp_json): _call(tmp_json, mgr, instance_id=0) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert all(r["annotation_id"] == 0 for r in data) assert len(data) == 2 # s1 and s2 both have annotation_id=0 def test_instance_id_list(self, mgr, tmp_json): _call(tmp_json, mgr, instance_id=[1, 2]) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert all(r["annotation_id"] in (1, 2) for r in data) def test_sample_and_instance_combined(self, mgr, tmp_json): _call(tmp_json, mgr, sample_id="s1", instance_id=1) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert len(data) == 1 assert data[0]["sample_id"] == "s1" assert data[0]["annotation_id"] == 1 def test_empty_result_when_no_match(self, mgr, tmp_json): _call(tmp_json, mgr, sample_id="nonexistent") - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert data == [] @@ -305,7 +320,7 @@ class TestWriteDataframeEmpty: def test_empty_df_writes_empty_json(self, tmp_json): mgr = _make_manager(df=pd.DataFrame()) _call(tmp_json, mgr) - data = json.loads(open(tmp_json).read()) + data = _records(tmp_json) assert data == [] def test_empty_df_writes_empty_csv(self, tmp_csv): diff --git a/tests/backend/test_write_history.py b/tests/backend/test_write_history.py index 3608fc70..be08367b 100644 --- a/tests/backend/test_write_history.py +++ b/tests/backend/test_write_history.py @@ -4,6 +4,7 @@ import os from unittest.mock import MagicMock, patch +import pandas as pd import pytest from weightslab.backend.logger import LoggerQueue @@ -54,6 +55,24 @@ def _call(path, lg_instance, **kwargs): return write_history(path, **kwargs) +def _load(path): + """Read a write_history JSON export back into the row-records shape + ({"global": [...], "sample": [...], "instance": [...]}) regardless of + which `orient` it was written with. + + write_history defaults to ``orient="columns"`` per section + (``{column: {row_index: value}}``) — compact, since it writes each + column name once per section instead of once per row. These tests + assert row-level behavior, so normalize back to records here. + """ + raw = json.loads(open(path, encoding="utf-8").read()) + return { + section: payload if isinstance(payload, list) + else pd.DataFrame(payload).to_dict(orient="records") + for section, payload in raw.items() + } + + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -80,44 +99,79 @@ def tmp_csv(tmp_path): class TestWriteHistoryJsonStructure: def test_json_has_all_sections(self, lg, tmp_json): _call(tmp_json, lg) - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert set(data.keys()) == {"global", "sample", "instance"} def test_global_section_not_empty(self, lg, tmp_json): _call(tmp_json, lg) - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert len(data["global"]) > 0 def test_sample_section_not_empty(self, lg, tmp_json): _call(tmp_json, lg) - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert len(data["sample"]) > 0 def test_instance_section_not_empty(self, lg, tmp_json): _call(tmp_json, lg) - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert len(data["instance"]) > 0 def test_global_row_keys(self, lg, tmp_json): _call(tmp_json, lg) - row = json.loads(open(tmp_json).read())["global"][0] + row = _load(tmp_json)["global"][0] assert {"graph_name", "experiment_hash", "step", "metric_value"} <= set(row.keys()) def test_sample_row_keys(self, lg, tmp_json): _call(tmp_json, lg) - row = json.loads(open(tmp_json).read())["sample"][0] + row = _load(tmp_json)["sample"][0] assert {"graph_name", "experiment_hash", "sample_id", "step", "metric_value"} <= set(row.keys()) def test_instance_row_keys(self, lg, tmp_json): _call(tmp_json, lg) - row = json.loads(open(tmp_json).read())["instance"][0] + row = _load(tmp_json)["instance"][0] assert { "graph_name", "experiment_hash", "sample_id", "annotation_id", "step", "metric_value" } <= set(row.keys()) + def test_default_orient_is_columnar(self, lg, tmp_json): + """Default orient="columns": each section is {column: {row_index: value}}, + not a list of row dicts — this is the whole point (one column-name + string per section instead of one per row).""" + _call(tmp_json, lg) + raw = json.loads(open(tmp_json, encoding="utf-8").read()) + assert isinstance(raw["global"], dict) + assert isinstance(raw["global"]["graph_name"], dict) + assert isinstance(raw["sample"], dict) + assert isinstance(raw["instance"], dict) + + def test_orient_records_reproduces_the_row_list_shape(self, lg, tmp_json): + _call(tmp_json, lg, orient="records") + raw = json.loads(open(tmp_json, encoding="utf-8").read()) + assert isinstance(raw["global"], list) + assert {"graph_name", "experiment_hash", "step", "metric_value"} <= set(raw["global"][0].keys()) + + def test_columnar_default_is_smaller_than_records_for_many_rows(self, tmp_path): + # Enough rows that the per-row column-name repetition in orient="records" + # dominates file size — the case write_history's history logs are + # actually shaped like (few columns, many steps). + many_lg = LoggerQueue(register=False) + many_lg.chkpt_manager = _mock_chkpt("h1") + for step in range(1, 201): + many_lg.add_scalars("loss", {"loss": float(step)}, step, + signal_per_sample=None, aggregate_by_step=False) + + cols_path = str(tmp_path / "cols.json") + recs_path = str(tmp_path / "recs.json") + with patch("weightslab.src.get_logger", return_value=many_lg): + write_history(cols_path, type_of_history="global") + write_history(recs_path, type_of_history="global", orient="records") + + assert os.path.getsize(cols_path) < os.path.getsize(recs_path) + def test_file_is_valid_json(self, lg, tmp_json): _call(tmp_json, lg) - json.loads(open(tmp_json).read()) # must not raise + _load(tmp_json) # must not raise def test_output_file_created(self, lg, tmp_json): _call(tmp_json, lg) @@ -131,38 +185,38 @@ def test_output_file_created(self, lg, tmp_json): class TestWriteHistoryJsonTypeFilter: def test_type_all_explicit(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="all") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert set(data.keys()) == {"global", "sample", "instance"} def test_type_none_means_all(self, lg, tmp_json): _call(tmp_json, lg, type_of_history=None) - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert set(data.keys()) == {"global", "sample", "instance"} def test_type_global_only(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert set(data.keys()) == {"global"} assert "sample" not in data assert "instance" not in data def test_type_sample_only(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="sample") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert set(data.keys()) == {"sample"} assert "global" not in data assert "instance" not in data def test_type_instance_only(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="instance") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert set(data.keys()) == {"instance"} assert "global" not in data assert "sample" not in data def test_type_instances_plural_alias(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="instances") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert set(data.keys()) == {"instance"} @@ -173,32 +227,32 @@ def test_type_instances_plural_alias(self, lg, tmp_json): class TestWriteHistoryJsonGraphFilter: def test_graph_name_filters_global(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", graph_name="loss") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["graph_name"] == "loss" for r in data["global"]) def test_graph_name_filters_sample(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="sample", graph_name="acc") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["graph_name"] == "acc" for r in data["sample"]) def test_graph_name_filters_instance(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="instance", graph_name="iou") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["graph_name"] == "iou" for r in data["instance"]) def test_unknown_graph_name_global_empty(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", graph_name="no_such_graph") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert data["global"] == [] def test_unknown_graph_name_sample_empty(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="sample", graph_name="no_such_graph") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert data["sample"] == [] def test_multiple_graphs_all_present_without_filter(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) graph_names = {r["graph_name"] for r in data["global"]} assert "loss" in graph_names and "acc" in graph_names @@ -210,28 +264,28 @@ def test_multiple_graphs_all_present_without_filter(self, lg, tmp_json): class TestWriteHistoryJsonHashFilter: def test_exp_hash_filters_global(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", experiment_hash="h1") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["experiment_hash"] == "h1" for r in data["global"]) def test_exp_hash_filters_sample(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="sample", experiment_hash="h2") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["experiment_hash"] == "h2" for r in data["sample"]) assert len(data["sample"]) > 0 def test_exp_hash_missing_global_empty(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", experiment_hash="hX") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert data["global"] == [] def test_exp_hash_filters_instance(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="instance", experiment_hash="h1") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["experiment_hash"] == "h1" for r in data["instance"]) def test_both_hashes_present_with_all(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", graph_name="loss", experiment_hash="all") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) hashes = {r["experiment_hash"] for r in data["global"]} assert "h1" in hashes and "h2" in hashes @@ -243,24 +297,24 @@ def test_both_hashes_present_with_all(self, lg, tmp_json): class TestWriteHistoryJsonSampleFilter: def test_sample_id_filters_sample_history(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="sample", graph_name="loss", sample_id="s1") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["sample_id"] == "s1" for r in data["sample"]) def test_sample_id_filters_instance_history(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="instance", sample_id="s1") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["sample_id"] == "s1" for r in data["instance"]) def test_instance_id_filters_instance_history(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="instance", instance_id=1) - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["annotation_id"] == 1 for r in data["instance"]) def test_sample_and_instance_id_combined(self, lg, tmp_json): # annotation_id=1 lives under h1 _call(tmp_json, lg, type_of_history="instance", sample_id="s1", instance_id=1, experiment_hash="h1") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert len(data["instance"]) == 1 assert data["instance"][0]["sample_id"] == "s1" assert data["instance"][0]["annotation_id"] == 1 @@ -268,7 +322,7 @@ def test_sample_and_instance_id_combined(self, lg, tmp_json): def test_sample_id_no_effect_on_global(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", graph_name="loss", sample_id="s1", experiment_hash="all") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) hashes = {r["experiment_hash"] for r in data["global"]} assert "h1" in hashes and "h2" in hashes @@ -280,14 +334,14 @@ def test_sample_id_no_effect_on_global(self, lg, tmp_json): class TestWriteHistoryJsonValues: def test_global_steps_present(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", graph_name="loss", experiment_hash="h1") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) steps = {r["step"] for r in data["global"]} assert 1 in steps and 2 in steps def test_sample_value_correct(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="sample", graph_name="loss", experiment_hash="h1", sample_id="s1") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert len(data["sample"]) == 1 assert abs(data["sample"][0]["metric_value"] - 1.0) < 1e-5 @@ -295,12 +349,12 @@ def test_instance_value_correct(self, lg, tmp_json): # value 0.8 for annotation_id=1 is stored under h1 _call(tmp_json, lg, type_of_history="instance", graph_name="iou", sample_id="s1", instance_id=1, experiment_hash="h1") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert abs(data["instance"][0]["metric_value"] - 0.8) < 1e-4 def test_combined_graph_hash_filter(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", graph_name="loss", experiment_hash="h2") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) for row in data["global"]: assert row["graph_name"] == "loss" assert row["experiment_hash"] == "h2" @@ -383,7 +437,7 @@ def test_empty_logger_writes_empty_sections_json(self, tmp_json): empty_lg.chkpt_manager = None with patch("weightslab.src.get_logger", return_value=empty_lg): write_history(tmp_json, experiment_hash="all") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert data["global"] == [] assert data["sample"] == [] assert data["instance"] == [] @@ -399,12 +453,12 @@ def test_empty_logger_writes_header_only_csv(self, tmp_csv): def test_case_insensitive_type(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="SAMPLE") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert set(data.keys()) == {"sample"} def test_case_insensitive_format(self, lg, tmp_json): _call(tmp_json, lg, format="JSON") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert "global" in data def test_returns_written_path(self, lg, tmp_json): @@ -455,14 +509,14 @@ def test_directory_path_csv_format(self, lg, tmp_path): def test_graph_name_list_filters_global(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", graph_name=["loss", "acc"]) - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["graph_name"] in {"loss", "acc"} for r in data["global"]) names = {r["graph_name"] for r in data["global"]} assert "loss" in names and "acc" in names def test_graph_name_list_filters_sample(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="sample", graph_name=["loss", "acc"]) - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) names = {r["graph_name"] for r in data["sample"]} assert "loss" in names and "acc" in names @@ -470,28 +524,28 @@ def test_sample_id_list_filters_sample(self, lg, tmp_json): # s2 for "loss" is under h1 only; use "all" to see both hashes _call(tmp_json, lg, type_of_history="sample", graph_name="loss", sample_id=["s1", "s2"], experiment_hash="all") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) sids = {r["sample_id"] for r in data["sample"]} assert sids == {"s1", "s2"} def test_sample_id_list_filters_instance(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="instance", sample_id=["s1", "s2"], experiment_hash="all") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) sids = {r["sample_id"] for r in data["instance"]} assert sids == {"s1", "s2"} def test_instance_id_list_filters_instance(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="instance", instance_id=[1, 2], experiment_hash="all") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) aids = {r["annotation_id"] for r in data["instance"]} assert aids == {1, 2} def test_instance_id_list_single_value(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="instance", instance_id=[1], experiment_hash="all") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) assert all(r["annotation_id"] == 1 for r in data["instance"]) # -- new: experiment_hash default / "all" behavior -- @@ -499,14 +553,14 @@ def test_instance_id_list_single_value(self, lg, tmp_json): def test_default_hash_uses_current(self, lg, tmp_json): # current hash is h2; without specifying, only h2 rows returned _call(tmp_json, lg, type_of_history="global", graph_name="loss") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) hashes = {r["experiment_hash"] for r in data["global"]} assert hashes == {"h2"} def test_experiment_hash_all_returns_every_hash(self, lg, tmp_json): _call(tmp_json, lg, type_of_history="global", graph_name="loss", experiment_hash="all") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) hashes = {r["experiment_hash"] for r in data["global"]} assert "h1" in hashes and "h2" in hashes @@ -514,7 +568,7 @@ def test_explicit_hash_overrides_current(self, lg, tmp_json): # pass h1 explicitly even though current is h2 _call(tmp_json, lg, type_of_history="global", graph_name="loss", experiment_hash="h1") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) hashes = {r["experiment_hash"] for r in data["global"]} assert hashes == {"h1"} @@ -526,7 +580,7 @@ def test_default_hash_no_chkpt_manager_returns_none_key(self, tmp_json): signal_per_sample=None, aggregate_by_step=False) with patch("weightslab.src.get_logger", return_value=lg): write_history(tmp_json, type_of_history="global") - data = json.loads(open(tmp_json).read()) + data = _load(tmp_json) # row exists (experiment_hash stored as "") assert len(data["global"]) == 1 assert data["global"][0]["experiment_hash"] == "" diff --git a/tests/components/test_checkpoint_workflow.py b/tests/components/test_checkpoint_workflow.py index 671676ca..6bbef8e9 100644 --- a/tests/components/test_checkpoint_workflow.py +++ b/tests/components/test_checkpoint_workflow.py @@ -308,7 +308,7 @@ def setUpClass(cls): cls.temp_dir = tempfile.mkdtemp(prefix="checkpoint_v3_test_") cls.log_dir = os.path.join(cls.temp_dir, "experiments") - # Initialize config from YAML-like dict (similar to ws-classification) + # Initialize config from YAML-like dict (similar to wl-classification) cls.config = { 'experiment_name': EXP_NAME, 'device': DEVICE, @@ -1298,33 +1298,15 @@ def test_logger_queue_saved_with_weights(self): self.chkpt_manager.update_experiment_hash(force=False, dump_immediately=False) self.chkpt_manager.save_pending_changes(force=True) - snapshot_dir = Path(self.chkpt_manager.loggers_dir) - manifest_path = snapshot_dir / "loggers.manifest.json" - legacy_snapshot_path = snapshot_dir / "loggers.json" - self.assertTrue( - manifest_path.exists() or legacy_snapshot_path.exists(), - "Logger snapshot should be saved with checkpoint" - ) - - if manifest_path.exists(): - with open(manifest_path, "r") as f: - manifest = json.load(f) - chunk_files = manifest.get("chunks", []) - self.assertGreaterEqual(len(chunk_files), 1, "Chunked logger snapshot should contain at least one chunk") - self.assertTrue( - all((snapshot_dir / chunk_name).exists() for chunk_name in chunk_files), - "All logger snapshot chunks referenced by manifest should exist" - ) - self.assertTrue(self.chkpt_manager.load_logger_snapshot()) - lg = ledgers.get_logger('main') - loggers = {'main': {'signal_history': lg.get_signal_history() if hasattr(lg, 'get_signal_history') else []}} - else: - with open(legacy_snapshot_path, "r") as f: - snapshot = json.load(f) - loggers = snapshot.get("loggers", {}) + # Logger history is persisted to an on-disk DuckDB file alongside the + # checkpoint (the chunked-zstd snapshot was replaced by DuckDB in #257). + db_path = Path(self.chkpt_manager.get_logger_db_path()) + self.assertTrue(db_path.exists(), "Logger DuckDB should be saved with checkpoint") - self.assertIn('main', loggers, "Logger entry should be present") - signals = loggers['main'].get("signal_history", []) + # The registered logger carries the persisted signal history. + lg = ledgers.get_logger('main') + self.assertIsNotNone(lg, "Logger entry should be present") + signals = lg.get_signal_history() if hasattr(lg, 'get_signal_history') else [] if isinstance(signals, dict): total_signals = 0 for experiments in signals.values(): diff --git a/tests/gRPC/test_grpc_user_actions.py b/tests/gRPC/test_grpc_user_actions.py index 1b71da74..530d8e7d 100644 --- a/tests/gRPC/test_grpc_user_actions.py +++ b/tests/gRPC/test_grpc_user_actions.py @@ -61,7 +61,7 @@ def get_combined_df(self): def get_df_view(self): return self.df.reset_index().copy()\ - def get_collapse_annotations_to_samples_df(self): + def get_collapse_annotations_to_samples_df(self, df=None): # This fixture is already one row per sample (no instance rows), so the # per-sample collapsed view is just the frame with its index restored to # (origin, sample_id) columns — matching what the real manager returns. diff --git a/tests/general/test_cli.py b/tests/general/test_cli.py index 64347b72..a63801dc 100644 --- a/tests/general/test_cli.py +++ b/tests/general/test_cli.py @@ -8,6 +8,7 @@ import time import json import tempfile +import argparse from pathlib import Path from unittest.mock import MagicMock @@ -23,6 +24,7 @@ cli_serve, ) import weightslab.backend.cli as cli_backend +import weightslab.cli as wl_cli from weightslab.backend.ledgers import GLOBAL_LEDGER, Proxy @@ -518,6 +520,56 @@ def test_cli_serve_writes_discovery_file(self): cli_backend._server_thread = None +class TestUIStartPortResolution(unittest.TestCase): + """Tests for UI port resolution used by `weightslab start`.""" + + def setUp(self): + self._saved_env = { + k: os.environ.get(k) + for k in ("WL_LAST_UI_PORT", "WEIGHTSLAB_UI_PORT", "WEIGHTSLAB_EXPERIMENT_CONFIG") + } + for key in self._saved_env: + os.environ.pop(key, None) + + def tearDown(self): + for key, value in self._saved_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + def test_explicit_port_wins(self): + args = argparse.Namespace(port=61234, config=None) + port, source = wl_cli._resolve_ui_port(args) + self.assertEqual(port, 61234) + self.assertEqual(source, "--port") + + def test_config_port_wins_over_env(self): + with tempfile.TemporaryDirectory() as tmpdir: + cfg = Path(tmpdir) / "config.yaml" + cfg.write_text("ui_port: 61235\n", encoding="utf-8") + os.environ["WL_LAST_UI_PORT"] = "62000" + args = argparse.Namespace(port=None, config=str(cfg)) + + port, source = wl_cli._resolve_ui_port(args) + self.assertEqual(port, 61235) + self.assertEqual(source, "experiment config") + + def test_last_ui_port_env_fallback(self): + os.environ["WL_LAST_UI_PORT"] = "61236" + args = argparse.Namespace(port=None, config=None) + + port, source = wl_cli._resolve_ui_port(args) + self.assertEqual(port, 61236) + self.assertEqual(source, "WL_LAST_UI_PORT") + + def test_default_is_50051(self): + args = argparse.Namespace(port=None, config=None) + port, source = wl_cli._resolve_ui_port(args) + self.assertEqual(port, 50051) + self.assertEqual(source, "default") + + def run_tests(): """Run all CLI tests.""" # Create test suite @@ -529,6 +581,7 @@ def run_tests(): suite.addTests(loader.loadTestsFromTestCase(TestCLICommands)) suite.addTests(loader.loadTestsFromTestCase(TestCLIServer)) suite.addTests(loader.loadTestsFromTestCase(TestCLIEndpointDiscovery)) + suite.addTests(loader.loadTestsFromTestCase(TestUIStartPortResolution)) # suite.addTests(loader.loadTestsFromTestCase(TestCLIIntegration)) # Run tests diff --git a/tests/general/test_logger_snapshot_rotation.py b/tests/general/test_logger_snapshot_rotation.py deleted file mode 100644 index f0a0001e..00000000 --- a/tests/general/test_logger_snapshot_rotation.py +++ /dev/null @@ -1,102 +0,0 @@ -import json -import tempfile -import unittest -from pathlib import Path - -from weightslab.backend import ledgers -from weightslab.components.checkpoint_manager import CheckpointManager -from weightslab.backend.logger import LoggerQueue - - -class LoggerSnapshotRotationTests(unittest.TestCase): - def setUp(self): - ledgers.clear_all() - - def tearDown(self): - ledgers.clear_all() - - def test_save_logger_snapshot_writes_chunked_zstd_and_can_reload(self): - with tempfile.TemporaryDirectory(prefix="weightslab_logger_snapshot_") as temp_dir: - manager = CheckpointManager(root_log_dir=temp_dir, load_model=False, load_config=False, load_data=False) - exp_hash = "12345678abcdef0199aabbcc" - manager.current_exp_hash = exp_hash - - logger_queue = LoggerQueue(register=False) - logger_queue.load_snapshot( - { - "graph_names": ["train/loss"], - "signal_history": [ - { - "experiment_name": "train/loss", - "model_age": 1, - "metric_name": "train/loss", - "metric_value": 0.42, - "experiment_hash": exp_hash, - } - ], - "signal_history_per_sample": {}, - } - ) - ledgers.register_logger(logger_queue, name="main") - - saved_path = manager.save_logger_snapshot() - self.assertIsNotNone(saved_path, "save_logger_snapshot should return a saved path") - self.assertTrue(saved_path.exists(), "Logger snapshot manifest should exist") - - snapshot_dir = Path(manager.loggers_dir) - manifest_path = snapshot_dir / "loggers.manifest.json" - self.assertTrue(manifest_path.exists(), "Manifest should be written for chunked snapshot format") - - with open(manifest_path, "r", encoding="utf-8") as f: - manifest = json.load(f) - chunk_names = manifest.get("chunks", []) - self.assertGreaterEqual(len(chunk_names), 1, "At least one compressed chunk should be created") - for chunk_name in chunk_names: - self.assertTrue((snapshot_dir / chunk_name).exists(), f"Missing expected chunk file: {chunk_name}") - - ledgers.clear_all() - loaded = manager.load_logger_snapshot() - self.assertTrue(loaded, "Chunked snapshot should load successfully") - restored_logger = ledgers.get_logger("main") - self.assertTrue(hasattr(restored_logger, "get_signal_history"), "Restored logger should support signal history") - self.assertGreaterEqual(len(restored_logger.get_signal_history()), 1, "Restored logger should contain signals") - - def test_load_logger_snapshot_supports_legacy_json(self): - with tempfile.TemporaryDirectory(prefix="weightslab_logger_snapshot_legacy_") as temp_dir: - manager = CheckpointManager(root_log_dir=temp_dir, load_model=False, load_config=False, load_data=False) - exp_hash = "abcdef0199aabbcc12345678" - manager.current_exp_hash = exp_hash - - snapshot_dir = Path(manager.loggers_dir) - snapshot_dir.mkdir(parents=True, exist_ok=True) - legacy_payload = { - "exp_hash": exp_hash, - "timestamp": "2026-02-24T00:00:00", - "loggers": { - "main": { - "graph_names": ["train/acc"], - "signal_history": [ - { - "experiment_name": "train/acc", - "model_age": 2, - "metric_name": "train/acc", - "metric_value": 0.9, - "experiment_hash": exp_hash, - } - ], - "signal_history_per_sample": {}, - } - }, - } - with open(snapshot_dir / "loggers.json", "w", encoding="utf-8") as f: - json.dump(legacy_payload, f) - - loaded = manager.load_logger_snapshot() - self.assertTrue(loaded, "Legacy JSON logger snapshot should still load") - restored_logger = ledgers.get_logger("main") - self.assertTrue(hasattr(restored_logger, "get_signal_history"), "Restored logger should support signal history") - self.assertGreaterEqual(len(restored_logger.get_signal_history()), 1, "Legacy snapshot should restore signals") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/general/test_signal_refinements.py b/tests/general/test_signal_refinements.py new file mode 100644 index 00000000..681c9970 --- /dev/null +++ b/tests/general/test_signal_refinements.py @@ -0,0 +1,163 @@ +"""Unit tests for the signal-refinements changes: per-sample query cache, +value-at-step read + staging fast path, cycle detection, ctx.logits, freshness, +and the reactive-derived gather skip.""" +import unittest + +import numpy as np +import torch + +import weightslab as wl +from weightslab.backend.logger import LoggerQueue +from weightslab.src import ( + _REGISTERED_SIGNALS, _detect_signal_cycles, _gather_inputs_fresh, + BatchSignalContext, SignalContext, StaleSignalError, +) + + +def _lg(): + return LoggerQueue(register=False) + + +class TestQueryCache(unittest.TestCase): + def test_cached_and_fresh_copy(self): + lg = _lg() + lg._stage_sample_row("m", "h", "1", 0, 1.5) + lg._stage_sample_row("m", "h", "2", 0, 2.5) + r1 = lg.query_per_sample("m", sample_ids=["1", "2"]) + r2 = lg.query_per_sample("m", sample_ids=["1", "2"]) + self.assertEqual(len(r1), 2) + self.assertIsNot(r1, r2) # fresh list each call + self.assertGreaterEqual(lg._qps_cache.cache_info().hits, 1) # 2nd read hit + + def test_invalidated_on_write(self): + lg = _lg() + lg._stage_sample_row("m", "h", "1", 0, 1.0) + self.assertEqual(len(lg.query_per_sample("m", sample_ids=["1"])), 1) + lg._stage_sample_row("m", "h", "1", 1, 9.0) # new row bumps version + self.assertEqual(len(lg.query_per_sample("m", sample_ids=["1"])), 2) + + def test_step_scoped_clear(self): + lg = _lg() + lg._stage_sample_row("m", "h", "1", 0, 1.0) + lg.query_per_sample("m", sample_ids=["1"]) + lg._stage_sample_row("m", "h", "2", 1, 2.0) # step advance -> clear + self.assertEqual(lg._qps_cache_step, 1) + + +class TestValueAtStep(unittest.TestCase): + def test_staging_fast_path(self): + lg = _lg() + lg._stage_sample_row("m", "h", "5", 3, 4.0) # staged, not flushed + at = dict((int(s), v) for s, v in lg.query_per_sample_at_step("m", [5], 3)) + self.assertEqual(at, {5: 4.0}) + + def test_absent_at_other_step(self): + lg = _lg() + lg._stage_sample_row("m", "h", "5", 3, 4.0) + self.assertEqual(lg.query_per_sample_at_step("m", [5], 99), []) + + +class TestCycleDetection(unittest.TestCase): + def setUp(self): _REGISTERED_SIGNALS.clear() + def tearDown(self): _REGISTERED_SIGNALS.clear() + + def test_self_loop_and_two_cycle(self): + @wl.signal(name="sig/self", inputs=["sig/self"], batched=True) + def _s(b): return b + @wl.signal(name="sig/X", inputs=["sig/Y"], batched=True) + def _x(b): return b + @wl.signal(name="sig/Y", inputs=["sig/X"], batched=True) + def _y(b): return b + keys = {frozenset(c) for c in _detect_signal_cycles()} + self.assertIn(frozenset(["sig/self"]), keys) + self.assertIn(frozenset(["sig/X", "sig/Y"]), keys) + + def test_dag_has_no_cycle(self): + @wl.signal(name="sig/a", inputs=["train/loss"], batched=True) # base leaf + def _a(b): return b + @wl.signal(name="sig/b", inputs=["sig/a"], batched=True) + def _b(b): return b + self.assertEqual(_detect_signal_cycles(), []) + + +class TestContexts(unittest.TestCase): + def test_batch_context_carries_logits(self): + b = BatchSignalContext(sample_ids=[1, 2], subscribed_values=[0.1, 0.2], + logits=torch.tensor([[1., 2., 3.], [4., 5., 6.]])) + self.assertEqual(tuple(b.logits.shape), (2, 3)) + + def test_sample_context_carries_logits(self): + c = SignalContext(sample_id=1, dataframe=None, logits=torch.tensor([1., 2., 3.])) + self.assertEqual(tuple(c.logits.shape), (3,)) + + def test_stale_signal_raises(self): + b = BatchSignalContext(sample_ids=[1], subscribed_values=[0.0], logger=_lg(), step=5) + with self.assertRaises(StaleSignalError): + b.latest("sig/never", require_fresh=True) + + +class TestGatherFreshCache(unittest.TestCase): + def setUp(self): _REGISTERED_SIGNALS.clear() + def tearDown(self): _REGISTERED_SIGNALS.clear() + + def test_reactive_derived_not_queried_when_absent(self): + @wl.signal(name="sig/derived", inputs=["x"], batched=True) # reactive + def _d(b): return b + lg = _lg() + calls = {"n": 0} + orig = lg.query_per_sample_at_step + lg.query_per_sample_at_step = lambda *a, **k: (calls.__setitem__("n", calls["n"] + 1), orig(*a, **k))[1] + res = _gather_inputs_fresh(lg, ["sig/derived"], [1], 0, fresh_cache={}) + self.assertIsNone(res) # not fired yet -> skip + self.assertEqual(calls["n"], 0) # and NOT a ledger query (no flush) + + def test_reactive_derived_read_from_cache(self): + @wl.signal(name="sig/derived", inputs=["x"], batched=True) + def _d(b): return b + res = _gather_inputs_fresh(_lg(), ["sig/derived"], [1, 2], 0, + fresh_cache={"sig/derived": np.array([1.0, 2.0])}) + self.assertIsNotNone(res) + np.testing.assert_array_equal(res["sig/derived"], [1.0, 2.0]) + + +class TestDefaultShapeClassifier(unittest.TestCase): + def test_labels(self): + self.assertEqual(wl.classify_loss_shape([2, 1.5, 1, 0.5, 0.05]), "monotonic") + self.assertEqual(wl.classify_loss_shape([2, 2, 2, 2, 2]), "Flat_high") + self.assertEqual(wl.classify_loss_shape([0.1, 0.1, 0.1, 3.0, 0.1]), "Spiked") + + def test_too_short_is_none(self): + self.assertIsNone(wl.classify_loss_shape([1, 2, 3])) + self.assertIsNone(wl.classify_loss_shape([1, 0.5, 0.1], min_points=5)) + + def test_thresholds_configurable(self): + traj = [1.0, 0.9, 0.8, 0.75, 0.7] # net drop 0.3 + self.assertNotEqual(wl.classify_loss_shape(traj), "monotonic") # default 0.4 unmet + self.assertEqual(wl.classify_loss_shape(traj, drop_learned=0.25), "monotonic") + + def test_trajectory_stats_reusable(self): + s = wl.trajectory_stats([2.0, 1.0, 0.5, 0.5, 0.4]) + self.assertAlmostEqual(s["drop"], (2.0 - 0.4) / 2.0, places=5) + self.assertIn("cv", s); self.assertIn("tail_cv", s); self.assertIn("argmin_frac", s) + self.assertIsNone(wl.trajectory_stats([1.0])) # < 2 points + + def test_enable_live_signal_registers(self): + _REGISTERED_SIGNALS.clear() + try: + wl.enable_loss_shape_signal(loss_signal="loss_sample", name="sig/live_shape", every=5) + self.assertIn("sig/live_shape", _REGISTERED_SIGNALS) + meta = _REGISTERED_SIGNALS["sig/live_shape"]._wl_signal_meta + self.assertEqual(meta.get("inputs"), ["loss_sample"]) + self.assertEqual(meta.get("compute_every_n_steps"), 5) + finally: + _REGISTERED_SIGNALS.clear() + + def test_exports(self): + for name in ("classify_loss_shape", "trajectory_stats", "write_loss_shapes", + "write_signal_shapes", "enable_loss_shape_signal"): + self.assertTrue(hasattr(wl, name), name) + self.assertIn("monotonic", wl.LOSS_SHAPES) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/general/test_signals.py b/tests/general/test_signals.py index 0d32127d..f9281417 100644 --- a/tests/general/test_signals.py +++ b/tests/general/test_signals.py @@ -4,15 +4,20 @@ import numpy as np from unittest.mock import MagicMock, patch import weightslab as wl -from weightslab.src import _REGISTERED_SIGNALS, compute_signals, wrappered_fwd +from weightslab.src import ( + _REGISTERED_SIGNALS, _REACTIVE_FIRED, _react_dependents, + SignalContext, BatchSignalContext, compute_signals, wrappered_fwd, +) class TestSignals(unittest.TestCase): def setUp(self): # Clear registered signals before each test _REGISTERED_SIGNALS.clear() + _REACTIVE_FIRED.clear() def tearDown(self): _REGISTERED_SIGNALS.clear() + _REACTIVE_FIRED.clear() # ========================================================================= # Input Type Tests - Different signal and data input formats @@ -251,6 +256,70 @@ def dynamic_sig(sample_id, value, dataframe=None, origin="train"): losses_data = call_args[1]['losses'] self.assertIn('signals//dynamic_sig', losses_data) + def test_context_exposes_inputs_attribute(self): + """SignalContext / BatchSignalContext always expose an `inputs` dict.""" + # Defaults to an empty dict (never AttributeError, even off the reactive path). + sctx = SignalContext(sample_id=1, dataframe=None) + self.assertEqual(sctx.inputs, {}) + bctx = BatchSignalContext(sample_ids=[1, 2], subscribed_values=[0.1, 0.2]) + self.assertEqual(bctx.inputs, {}) + + # Populated via the constructor (as the reactive dispatch does). + cols = {"loss_sample": np.array([1.0, 2.0]), "sig/entropy": np.array([0.3, 0.4])} + bctx2 = BatchSignalContext(sample_ids=[1, 2], subscribed_values=cols["loss_sample"], inputs=cols) + self.assertIn("loss_sample", bctx2.inputs) + self.assertIn("sig/entropy", bctx2.inputs) + np.testing.assert_array_equal(bctx2.inputs["loss_sample"], [1.0, 2.0]) + + def test_reactive_inputs_populated_and_chained(self): + """A @wl.signal(inputs=[...]) reads its inputs via ctx.inputs[name], and a + derived input (another signal) is visible to a downstream signal.""" + class FakeLogger: + def __init__(self, data): # {sig: {sid: {step: val}}} + self.d = data + def query_per_sample_at_step(self, name, ids, step): + return [(sid, self.d[name][sid][step]) for sid in ids + if name in self.d and sid in self.d[name] + and step in self.d[name][sid]] + def query_per_sample(self, name, sample_ids=None): + rows = [] + for sid in (sample_ids or []): + for st, v in sorted(self.d.get(name, {}).get(sid, {}).items()): + rows.append((sid, st, v, None)) + return rows + + captured = {} + + @wl.signal(name="sig/loss_norm", inputs=["loss_sample"], batched=True) + def loss_norm(b): + captured["norm"] = dict(b.inputs) + return b.inputs["loss_sample"] / (float(np.mean(b.inputs["loss_sample"])) + 1e-8) + + @wl.signal(name="sig/hardness", inputs=["loss_sample", "sig/loss_norm"], batched=True) + def hardness(b): + captured["hard"] = dict(b.inputs) + return b.inputs["loss_sample"] * b.inputs["sig/loss_norm"] + + fake = FakeLogger({"loss_sample": {1: {10: 2.0}, 2: {10: 4.0}}}) + saved = {} + + def fake_save_signals(signals=None, **kw): + saved.update(signals or {}) + + with patch("weightslab.src.get_logger", return_value=fake), \ + patch("weightslab.src.get_dataframe", return_value=None), \ + patch("weightslab.src.save_signals", side_effect=fake_save_signals): + _react_dependents(["loss_sample"], batch_ids=[1, 2], step=10, origin="train") + + # loss_norm read its base-metric input through ctx.inputs + np.testing.assert_array_equal(captured["norm"]["loss_sample"], [2.0, 4.0]) + # hardness saw BOTH the base metric AND the derived signal via ctx.inputs + self.assertIn("loss_sample", captured["hard"]) + self.assertIn("sig/loss_norm", captured["hard"]) + # Both fired and were persisted + self.assertIn("sig/loss_norm", saved) + self.assertIn("sig/hardness", saved) + def test_frequency_control(self): """Test compute_every_n_steps.""" with patch("weightslab.src.list_models", return_value=["main"]), \ diff --git a/tests/integrations/ultralytics/ddp/ddp_ablation.py b/tests/integrations/ultralytics/ddp/ddp_ablation.py index 3c268e84..156762a8 100644 --- a/tests/integrations/ultralytics/ddp/ddp_ablation.py +++ b/tests/integrations/ultralytics/ddp/ddp_ablation.py @@ -31,10 +31,10 @@ import torch.distributed as dist import torch.multiprocessing as mp -# This harness lives under tests/ but drives the ws-detection usecase: put the usecase +# This harness lives under tests/ but drives the wl-detection usecase: put the usecase # src on the path (for yolo_pipeline / utils.*) and resolve config/data/ddp_run vs IT. _USECASE_SRC = os.path.abspath(os.path.join( - os.path.dirname(__file__), "../../../../examples/PyTorch/ws-detection/src")) + os.path.dirname(__file__), "../../../../examples/PyTorch/wl-detection/src")) sys.path.insert(0, _USECASE_SRC) _HERE = _USECASE_SRC _HOST = "127.0.0.1" @@ -149,7 +149,9 @@ def _worker(rank, world, master_port): _ensure_core_ddp_registered, reconcile_all, flush_outbox) _ensure_core_ddp_registered() import weightslab as wl - wl.serve(serving_grpc=True, serving_cli=False) + # DDP ablation harness: config wrapping is driven by yolo_pipeline across + # ranks, so opt out of the serve() serving-config guard explicitly. + wl.serve(serving_grpc=True, serving_cli=False, allow_unconfigured=True) decode = yolo_pipeline._decode_preds_to_6col else: model, loader, crit, iou, optimizer = _build_ul(cfg, device, batch_size, num_workers) diff --git a/tests/integrations/ultralytics/ddp/ddp_test_suite.py b/tests/integrations/ultralytics/ddp/ddp_test_suite.py index c9608081..280ef57e 100644 --- a/tests/integrations/ultralytics/ddp/ddp_test_suite.py +++ b/tests/integrations/ultralytics/ddp/ddp_test_suite.py @@ -40,10 +40,10 @@ import grpc import sys -# Harness lives under tests/ — put the ws-detection usecase src on the path so the +# Harness lives under tests/ — put the wl-detection usecase src on the path so the # usecase modules (yolo_pipeline, utils.*) and its config/data/ddp_run resolve. sys.path.insert(0, os.path.abspath(os.path.join( - os.path.dirname(__file__), "../../../../examples/PyTorch/ws-detection/src"))) + os.path.dirname(__file__), "../../../../examples/PyTorch/wl-detection/src"))) import yolo_pipeline # reuse _build_pipeline / _decode_preds_to_6col / _HERE / _LOSS_PARTS import weightslab.proto.experiment_service_pb2 as pb2 diff --git a/tests/model/test_constraint_generation.py b/tests/model/test_constraint_generation.py index ddd00a94..dcc8caf1 100644 --- a/tests/model/test_constraint_generation.py +++ b/tests/model/test_constraint_generation.py @@ -15,9 +15,48 @@ from weightslab.utils.modules_dependencies import DepType from weightslab.backend.model_interface import ModelInterface +from weightslab.backend import ledgers from weightslab.utils.computational_graph import _detect_layer_constraints +def _onnx_export_available() -> bool: + """True when the ONNX export toolchain imports cleanly. + + ``torch.onnx.export`` pulls in ``onnxscript``, which (as of 0.6.x) imports a + separately-packaged ``onnx_ir``. When that package is absent, importing + ``onnxscript`` raises ``ModuleNotFoundError: No module named 'onnx_ir'`` and + every ONNX-based dependency-extraction test fails at setup. Guard on this so + those tests skip where the toolchain is missing but still run once the + ``onnx``/dev extras are fully installed. + """ + try: + import onnxscript # noqa: F401 # transitively imports onnx_ir + return True + except Exception: + return False + + +_ONNX_EXPORT_AVAILABLE = _onnx_export_available() +_ONNX_SKIP_REASON = "ONNX export toolchain unavailable (onnxscript/onnx_ir not installed)" + + +class _RegistryIsolation: + """Clean the global model registry after each test. + + ``get_dependencies_onnx`` builds a throwaway ``ModelInterface`` (in ``eval`` + mode, for ONNX dependency extraction) which registers itself as the global + experiment model and is never cleaned up. Left behind, ``get_model()`` keeps + returning that eval-mode model, and anything that reads it — e.g. + ``LoggerQueue._get_audit_mode`` (``get_model().audit_mode``) — sees + ``audit_mode == True``, leaking into unrelated tests run later in the suite. + """ + def tearDown(self): + try: + ledgers.GLOBAL_LEDGER.unregister_model() + except Exception: + pass + + def get_dependencies_onnx(model: nn.Module, dummy_input: torch.Tensor) -> List[Tuple[nn.Module, nn.Module, DepType]]: """Extract dependencies using ONNX export""" try: @@ -96,8 +135,8 @@ def forward(self, x): return x -@unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") -class TestConstraintDetection(unittest.TestCase): +# @unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") +class TestConstraintDetection(_RegistryIsolation, unittest.TestCase): """Test constraint detection on individual modules""" def test_grouped_conv_detection(self): @@ -136,8 +175,9 @@ def test_batchnorm_no_constraint(self): self.assertEqual(len(constraints), 0) -@unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") -class TestConstraintPropagation(unittest.TestCase): +# @unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") +@unittest.skipUnless(_ONNX_EXPORT_AVAILABLE, _ONNX_SKIP_REASON) +class TestConstraintPropagation(_RegistryIsolation, unittest.TestCase): """Test constraint propagation through dependency graphs""" def test_grouped_conv_propagation(self): @@ -201,8 +241,9 @@ def test_multi_grouped_propagation(self): self.assertEqual(model.model.g_conv3.wl_constraints[1]['cons_group_size'], 8) -@unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") -class TestConstraintReporting(unittest.TestCase): +# @unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") +@unittest.skipUnless(_ONNX_EXPORT_AVAILABLE, _ONNX_SKIP_REASON) +class TestConstraintReporting(_RegistryIsolation, unittest.TestCase): """Test constraint reporting and information retrieval""" def test_constraint_source_tracking(self): diff --git a/tests/model/test_model_with_ops.py b/tests/model/test_model_with_ops.py index 827fb703..66935955 100644 --- a/tests/model/test_model_with_ops.py +++ b/tests/model/test_model_with_ops.py @@ -25,7 +25,7 @@ TMP_DIR = '/tmp/utests/'; os.makedirs('/tmp/utests/', exist_ok=True) -@unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") +# @unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") class NetworkWithOpsTest(unittest.TestCase): def setUp(self) -> None: print(f"\n--- Start {self._testMethodName} ---\n") @@ -206,8 +206,8 @@ def test_train_freeze(self): # Set Tracker self.dummy_network.set_tracking_mode(TrackingMode.TRAIN) - # Train for like 10 epochs - for _ in trange(10, desc="Training.."): + # Train for like 5 epochs + for _ in trange(5, desc="Training.."): self._train_one_epoch(cutoff=20) # Operate on the first layer - FREEZE diff --git a/tests/model/test_model_with_ops_unit.py b/tests/model/test_model_with_ops_unit.py index dba4df34..88a3d902 100644 --- a/tests/model/test_model_with_ops_unit.py +++ b/tests/model/test_model_with_ops_unit.py @@ -24,7 +24,7 @@ def __init__(self): self.l2 = _Layer(20) -@unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") +# @unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") class TestModelWithOpsUnit(unittest.TestCase): def test_reverse_index_and_tracking_mode(self): net = _Net() diff --git a/tests/model/test_tracking.py b/tests/model/test_tracking.py index 411730f5..33a789d7 100644 --- a/tests/model/test_tracking.py +++ b/tests/model/test_tracking.py @@ -148,7 +148,7 @@ def test_triggers_tracker_all_ops_plus_save_and_load(self): self.assertEqual(self.triggers_tracker, replica_triggers_tracker) -@unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") +# @unittest.skip("Constraint detection and propagation tests are currently skipped due to ongoing refactor and potential changes in the underlying implementation. Will be re-enabled once the new system is in place more modeling.") class TriggersTrackerClazzAndSampleIDTest(unittest.TestCase): """ Tests the TriggersTrackerClazzAndSampleID for the neuron operations and persistency. diff --git a/tests/test_secure_communication.py b/tests/test_secure_communication.py new file mode 100644 index 00000000..75c38f19 --- /dev/null +++ b/tests/test_secure_communication.py @@ -0,0 +1,132 @@ +"""Tests for secure environment setup on the native stack.""" + +import os +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from weightslab.security import CertAuthManager + + +class TestCertAuthManager: + """Certificate + auth token manager tests.""" + + def test_manager_init_with_custom_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + manager = CertAuthManager(certs_dir=tmpdir, enable_auth=True) + assert manager.certs_dir == Path(tmpdir) + assert manager.enable_auth is True + assert manager.cert_file == Path(tmpdir) / "backend-server.crt" + assert manager.key_file == Path(tmpdir) / "backend-server.key" + assert manager.ca_file == Path(tmpdir) / "ca.crt" + + def test_has_valid_certs_missing(self): + with tempfile.TemporaryDirectory() as tmpdir: + manager = CertAuthManager(certs_dir=tmpdir) + assert manager.has_valid_certs() is False + + def test_auth_token_generation(self): + with tempfile.TemporaryDirectory() as tmpdir: + manager = CertAuthManager(certs_dir=tmpdir, enable_auth=True) + token1 = manager.get_or_create_auth_token() + token2 = manager.get_or_create_auth_token() + assert token1 + assert token1 == token2 + + def test_auth_token_disabled(self): + with tempfile.TemporaryDirectory() as tmpdir: + manager = CertAuthManager(certs_dir=tmpdir, enable_auth=False) + assert manager.get_or_create_auth_token() == "" + + def test_tls_environment_variables_no_legacy_envoy_vars(self): + with tempfile.TemporaryDirectory() as tmpdir: + manager = CertAuthManager(certs_dir=tmpdir) + env_vars = manager.setup_tls_environment() + + assert env_vars["GRPC_TLS_ENABLED"] == "1" + assert env_vars["GRPC_TLS_REQUIRE_CLIENT_AUTH"] == "1" + assert "GRPC_TLS_CERT_FILE" in env_vars + assert "GRPC_TLS_KEY_FILE" in env_vars + assert "GRPC_TLS_CA_FILE" in env_vars + assert env_vars["WEIGHTSLAB_CERTS_DIR"] == tmpdir + + assert "ENVOY_DOWNSTREAM_TLS" not in env_vars + assert "ENVOY_UPSTREAM_TLS" not in env_vars + assert "VITE_SERVER_PROTOCOL" not in env_vars + + def test_auth_environment_variables_enabled(self): + with tempfile.TemporaryDirectory() as tmpdir: + manager = CertAuthManager(certs_dir=tmpdir, enable_auth=True) + env_vars = manager.setup_auth_environment() + + assert env_vars["WL_ENABLE_GRPC_AUTH_TOKEN"] == "1" + assert "GRPC_AUTH_TOKEN" in env_vars + assert "VITE_WL_ENABLE_GRPC_AUTH_TOKEN" not in env_vars + assert "VITE_GRPC_AUTH_TOKEN" not in env_vars + + def test_auth_environment_variables_disabled(self): + with tempfile.TemporaryDirectory() as tmpdir: + manager = CertAuthManager(certs_dir=tmpdir, enable_auth=False) + env_vars = manager.setup_auth_environment() + + assert env_vars["WL_ENABLE_GRPC_AUTH_TOKEN"] == "0" + assert "GRPC_AUTH_TOKEN" not in env_vars + + def test_from_env_or_default(self): + with tempfile.TemporaryDirectory() as tmpdir: + os.environ["WEIGHTSLAB_CERTS_DIR"] = tmpdir + try: + manager = CertAuthManager.from_env_or_default() + assert manager.certs_dir == Path(tmpdir) + finally: + del os.environ["WEIGHTSLAB_CERTS_DIR"] + + @patch("weightslab.security.cert_auth_manager.subprocess.run") + def test_generate_certs_success(self, mock_run): + mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") + with tempfile.TemporaryDirectory() as tmpdir: + manager = CertAuthManager(certs_dir=tmpdir) + success, _ = manager.generate_certs() + assert success is True + + @patch("weightslab.security.cert_auth_manager.subprocess.run") + @patch("weightslab.security.cert_auth_manager.Path.exists") + def test_generate_certs_script_missing(self, mock_exists, mock_run): + # First exists call is for cert file checks (False), then script path (False). + mock_exists.return_value = False + mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="") + + with tempfile.TemporaryDirectory() as tmpdir: + manager = CertAuthManager(certs_dir=tmpdir) + success, msg = manager.generate_certs() + assert success is False + assert "not found" in msg.lower() + + +class TestSecureInitialization: + """Import-time secure initialization sanity checks.""" + + def test_import_with_secure_init(self): + try: + import weightslab + assert weightslab is not None + except Exception as exc: + pytest.fail(f"Import failed: {exc}") + + def test_secure_init_skip_env_var(self): + os.environ["WEIGHTSLAB_SKIP_SECURE_INIT"] = "true" + try: + import importlib + import weightslab + + importlib.reload(weightslab) + except Exception as exc: + pytest.fail(f"Import failed: {exc}") + finally: + del os.environ["WEIGHTSLAB_SKIP_SECURE_INIT"] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_secure_docker.py b/tests/test_secure_docker.py deleted file mode 100644 index d7b9bd30..00000000 --- a/tests/test_secure_docker.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Tests for secure Docker setup with TLS and gRPC auth.""" - -import os -import tempfile -import pytest -from pathlib import Path -from unittest.mock import patch, MagicMock - -from weightslab.security import CertAuthManager -from weightslab.ui_docker_bridge import ( - _test_backend_connection, - _is_windows, -) - - -class TestCertAuthManager: - """Test certificate and auth token management.""" - - def test_manager_init_with_custom_dir(self): - """Test manager initialization with custom directory.""" - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir, enable_auth=True) - - assert manager.certs_dir == Path(tmpdir) - assert manager.enable_auth is True - assert manager.cert_file == Path(tmpdir) / 'backend-server.crt' - assert manager.key_file == Path(tmpdir) / 'backend-server.key' - assert manager.ca_file == Path(tmpdir) / 'ca.crt' - - def test_manager_init_with_default_dir(self): - """Test manager initialization with default directory.""" - manager = CertAuthManager(enable_auth=True) - - user_profile = os.environ.get('HOME') or os.path.expanduser('~') - expected_dir = Path(user_profile) / '.weightslab-certs' - - assert manager.certs_dir == expected_dir - - def test_has_valid_certs_missing(self): - """Test certificate validation when certs don't exist.""" - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir) - - assert manager.has_valid_certs() is False - - def test_auth_token_generation(self): - """Test gRPC auth token generation.""" - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir, enable_auth=True) - - token1 = manager.get_or_create_auth_token() - assert token1 is not None - assert len(token1) > 0 - assert isinstance(token1, str) - - # Should return same token on second call - token2 = manager.get_or_create_auth_token() - assert token1 == token2 - - def test_auth_token_disabled(self): - """Test that no token is generated when auth is disabled.""" - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir, enable_auth=False) - - token = manager.get_or_create_auth_token() - assert token == "" - - def test_tls_environment_variables(self): - """Test TLS environment variable setup.""" - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir) - - env_vars = manager.setup_tls_environment() - - assert 'GRPC_TLS_ENABLED' in env_vars - assert env_vars['GRPC_TLS_ENABLED'] == '1' - assert 'GRPC_TLS_CERT_FILE' in env_vars - assert 'GRPC_TLS_KEY_FILE' in env_vars - assert 'GRPC_TLS_CA_FILE' in env_vars - assert 'WEIGHTSLAB_CERTS_DIR' in env_vars - assert 'ENVOY_DOWNSTREAM_TLS' in env_vars - assert 'ENVOY_UPSTREAM_TLS' in env_vars - - def test_auth_environment_variables_enabled(self): - """Test auth environment variables when enabled.""" - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir, enable_auth=True) - - env_vars = manager.setup_auth_environment() - - assert 'WL_ENABLE_GRPC_AUTH_TOKEN' in env_vars - assert env_vars['WL_ENABLE_GRPC_AUTH_TOKEN'] == '1' - assert 'GRPC_AUTH_TOKEN' in env_vars - assert len(env_vars['GRPC_AUTH_TOKEN']) > 0 - - def test_auth_environment_variables_disabled(self): - """Test auth environment variables when disabled.""" - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir, enable_auth=False) - - env_vars = manager.setup_auth_environment() - - assert 'WL_ENABLE_GRPC_AUTH_TOKEN' in env_vars - assert env_vars['WL_ENABLE_GRPC_AUTH_TOKEN'] == '0' - assert 'GRPC_AUTH_TOKEN' not in env_vars - - def test_from_env_or_default(self): - """Test creating manager from environment variables.""" - with tempfile.TemporaryDirectory() as tmpdir: - os.environ['WEIGHTSLAB_CERTS_DIR'] = tmpdir - - try: - manager = CertAuthManager.from_env_or_default() - assert manager.certs_dir == Path(tmpdir) - finally: - del os.environ['WEIGHTSLAB_CERTS_DIR'] - - @patch('weightslab.security.cert_auth_manager.subprocess.run') - def test_generate_certs_success(self, mock_run): - """Test successful certificate generation.""" - mock_run.return_value = MagicMock(returncode=0, stdout='', stderr='') - - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir) - - # Since we're mocking subprocess, we need to create the files - manager.certs_dir.mkdir(parents=True, exist_ok=True) - manager.cert_file.touch() - manager.key_file.touch() - manager.ca_file.touch() - - success, msg = manager.generate_certs() - assert success is True - - @patch('weightslab.security.cert_auth_manager.subprocess.run') - def test_generate_certs_script_not_found(self, mock_run): - """Test certificate generation when script is missing.""" - mock_run.return_value = MagicMock(returncode=1, stdout='', stderr='Script not found') - - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir) - - success, msg = manager.generate_certs() - - # Should fail when script returns non-zero exit code - assert success is False or not manager.has_valid_certs() - - -class TestBackendConnection: - """Test backend connection utilities.""" - - def test_backend_connection_timeout(self): - """Test backend connection with timeout.""" - result = _test_backend_connection( - host='127.0.0.1', - port=59999, # Likely not listening - timeout=0.5 - ) - assert result is False - - @patch('socket.socket') - def test_backend_connection_success(self, mock_socket): - """Test successful backend connection.""" - mock_sock_instance = MagicMock() - mock_sock_instance.connect_ex.return_value = 0 - mock_socket.return_value = mock_sock_instance - - result = _test_backend_connection() - assert result is True - - @patch('socket.socket') - def test_backend_connection_failure(self, mock_socket): - """Test failed backend connection.""" - mock_sock_instance = MagicMock() - mock_sock_instance.connect_ex.return_value = 1 - mock_socket.return_value = mock_sock_instance - - result = _test_backend_connection() - assert result is False - - -class TestDockerBridgeIntegration: - """Integration tests for Docker bridge.""" - - def test_is_windows(self): - """Test Windows detection.""" - result = _is_windows() - assert isinstance(result, bool) - - def test_cert_auth_environment_integration(self): - """Test that cert auth manager sets env vars correctly.""" - with tempfile.TemporaryDirectory() as tmpdir: - manager = CertAuthManager(certs_dir=tmpdir, enable_auth=True) - - # Create dummy cert files - tmpdir_path = Path(tmpdir) - tmpdir_path.mkdir(parents=True, exist_ok=True) - (tmpdir_path / 'backend-server.crt').touch() - (tmpdir_path / 'backend-server.key').touch() - (tmpdir_path / 'ca.crt').touch() - - success, msg = manager.initialize() - - # Check that env vars were set - assert os.environ.get('GRPC_TLS_ENABLED') == '1' - assert os.environ.get('WL_ENABLE_GRPC_AUTH_TOKEN') == '1' - assert 'GRPC_AUTH_TOKEN' in os.environ - - -class TestSecureInitialization: - """Test secure initialization at package import.""" - - def test_import_with_secure_init(self): - """Test that secure init doesn't break import.""" - # This is a basic sanity check - try: - import weightslab - assert weightslab is not None - except Exception as e: - pytest.fail(f"Import failed: {e}") - - def test_secure_init_skip_env_var(self): - """Test that WEIGHTSLAB_SKIP_SECURE_INIT works.""" - os.environ['WEIGHTSLAB_SKIP_SECURE_INIT'] = 'true' - - try: - # Re-import should use skip flag - import importlib - import weightslab - importlib.reload(weightslab) - except Exception as e: - pytest.fail(f"Import failed: {e}") - finally: - del os.environ['WEIGHTSLAB_SKIP_SECURE_INIT'] - - -if __name__ == '__main__': - pytest.main([__file__, '-v']) diff --git a/tests/trainer/services/test_data_service_tabular_task_type.py b/tests/trainer/services/test_data_service_tabular_task_type.py new file mode 100644 index 00000000..7668c050 --- /dev/null +++ b/tests/trainer/services/test_data_service_tabular_task_type.py @@ -0,0 +1,98 @@ +"""Unit tests for tabular task_type auto-detection in data_service.py. + +Datasets whose model input is a 1-D feature vector (no image) should be +auto-detected as task_type "tabular" — distinct from "classification" — even +when the dataset/model never sets an explicit `task_type` attribute. See +`peek_is_tabular_sample` (used as the last step of the "Robust Task Type +Detection" heuristic in `DataService._process_sample_row`). +""" + +import unittest + +import numpy as np +import torch + +from weightslab.trainer.services.data_service import peek_is_tabular_sample + + +class _TabularDataset: + """Minimal stand-in for a tabular Dataset (ledger get_items contract). + + ``__getitem__`` is required only because ``load_raw_image_array`` uses its + presence as an "is this dataset-like" gate before consulting ``get_items``. + """ + + def __init__(self, n=8, n_features=16): + self.features = torch.randn(n, n_features) + + def __len__(self): + return len(self.features) + + def __getitem__(self, idx): + return self.features[idx], idx, 0 + + def get_index_from_sample_id(self, sample_id): + return sample_id if 0 <= sample_id < len(self.features) else None + + def get_items(self, idx, include_metadata=False, include_labels=False, include_images=False): + image = self.features[idx] if include_images else None + return image, idx, 0, None + + +class _ImageDataset: + """Minimal stand-in for a vision Dataset (H×W×C image input).""" + + def __init__(self, n=8): + self.images = torch.randint(0, 255, (n, 32, 32, 3), dtype=torch.uint8) + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + return self.images[idx], idx, 0 + + def get_index_from_sample_id(self, sample_id): + return sample_id if 0 <= sample_id < len(self.images) else None + + def get_items(self, idx, include_metadata=False, include_labels=False, include_images=False): + image = self.images[idx] if include_images else None + return image, idx, 0, None + + +class TestPeekIsTabularSample(unittest.TestCase): + def test_true_for_1d_feature_vector_input(self): + ds = _TabularDataset() + self.assertTrue(peek_is_tabular_sample(ds, 0)) + self.assertTrue(peek_is_tabular_sample(ds, 3)) + + def test_false_for_image_input(self): + ds = _ImageDataset() + self.assertFalse(peek_is_tabular_sample(ds, 0)) + + def test_false_when_sample_id_unresolvable(self): + ds = _TabularDataset(n=4) + self.assertFalse(peek_is_tabular_sample(ds, 999)) + + def test_false_on_dataset_without_expected_lookup_methods(self): + # No get_physical_location / get_index_from_sample_id / __getitem__ + # that behaves as expected -> best-effort heuristic, never raises. + class Broken: + pass + + self.assertFalse(peek_is_tabular_sample(Broken(), 0)) + + def test_respects_get_physical_location_when_present(self): + calls = [] + + class GroupedTabularDataset(_TabularDataset): + def get_physical_location(self, sample_id): + calls.append(sample_id) + return sample_id, 0 + + ds = GroupedTabularDataset() + self.assertTrue(peek_is_tabular_sample(ds, 2)) + self.assertEqual(calls, [2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/weightslab/__init__.py b/weightslab/__init__.py index e41340b6..06008e4a 100644 --- a/weightslab/__init__.py +++ b/weightslab/__init__.py @@ -10,13 +10,63 @@ import os import logging import threading +import importlib -from .src import watch_or_edit, start_training, serve, keep_serving, save_signals, save_instance_signals, save_group_signals, tag_samples, register_categorical_tag, set_categorical_tag, discard_samples, get_samples_by_tag, get_discarded_samples, signal, eval_fn, compute_signals, SignalContext, clear_all, run_pending_evaluation, trigger_pending_evaluation_async, query_signal_history, query_sample_history, query_instance_history, write_history, write_dataframe, get_current_experiment_hash, pointcloud_thumbnail, pointcloud_boxes -from .backend.ledgers import GLOBAL_LEDGER as ledger +# Only lightweight, torch-free helpers are imported eagerly (needed by the +# package-init side effects below). The banner and logging utilities pull in no +# heavy scientific stack. from .art import _BANNER from .utils.logs import setup_logging, set_log_directory, is_main_process -from .utils.tools import seed_everything -from .components.global_monitoring import guard_training_context, guard_testing_context + +# --- Lazy re-exports (PEP 562) --------------------------------------------- # +# The training API (`.src`, ledger, guards, seed_everything) transitively +# imports torch/numpy, which costs several seconds. Resolving these lazily means +# `import weightslab` — and in particular the `weightslab` CLI process, which +# only serves the UI and never touches a model — stays torch-free until the +# training helpers are actually used. A user's training script triggers the +# import the moment it first calls e.g. `wl.watch_or_edit(...)`, right where +# they'd import torch themselves anyway, so nothing changes for them. +_LAZY_EXPORTS = { + # ledger + guards + seeding live outside .src + "ledger": (".backend.ledgers", "GLOBAL_LEDGER"), + "seed_everything": (".utils.tools", "seed_everything"), + "guard_training_context": (".components.global_monitoring", "guard_training_context"), + "guard_testing_context": (".components.global_monitoring", "guard_testing_context"), +} +# Everything re-exported straight from .src (attribute name == export name). +for _name in ( + "watch_or_edit", "start_training", "serve", "keep_serving", "save_signals", + "save_instance_signals", "save_group_signals", "tag_samples", + "register_categorical_tag", "set_categorical_tag", "discard_samples", + "get_samples_by_tag", "get_discarded_samples", "signal", "eval_fn", + "compute_signals", "SignalContext", "BatchSignalContext", "StaleSignalError", + "drain_signals", "clear_all", "run_pending_evaluation", + "trigger_pending_evaluation_async", "query_signal_history", + "query_sample_history", "query_instance_history", "write_history", + "write_dataframe", "classify_loss_shape", "trajectory_stats", + "write_loss_shapes", "write_signal_shapes", "enable_loss_shape_signal", + "enable_loss_shape_autotag", "disable_loss_shape_autotag", + "auto_loss_shape_signal_names", + "LOSS_SHAPES", "get_current_experiment_hash", "pointcloud_thumbnail", + "pointcloud_boxes", +): + _LAZY_EXPORTS[_name] = (".src", _name) +del _name + + +def __getattr__(name): # PEP 562 module-level lazy attribute access + target = _LAZY_EXPORTS.get(name) + if target is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_name, attr = target + module = importlib.import_module(module_name, __name__) + value = getattr(module, attr) + globals()[name] = value # cache so __getattr__ isn't hit again + return value + + +def __dir__(): + return sorted(set(globals()) | set(_LAZY_EXPORTS)) # If you already have other top-level exports, keep them. # This snippet ensures __version__ is available even when setuptools_scm hasn't written the file yet. @@ -46,19 +96,19 @@ grpc_tls_enabled = os.environ.get('GRPC_TLS_ENABLED', 'true').lower() == 'true' if _IS_MAIN_PROCESS and grpc_tls_enabled and os.environ.get('WEIGHTSLAB_SKIP_SECURE_INIT', 'false').lower() != 'true': - try: - from weightslab.security import CertAuthManager + try: + from weightslab.security import CertAuthManager - enable_auth = os.environ.get('WL_ENABLE_GRPC_AUTH_TOKEN', 'true').lower() == 'true' - manager = CertAuthManager.from_env_or_default(enable_auth=enable_auth) + enable_auth = os.environ.get('WL_ENABLE_GRPC_AUTH_TOKEN', 'true').lower() == 'true' + manager = CertAuthManager.from_env_or_default(enable_auth=enable_auth) - success, msg = manager.check_and_apply() - if success: - logger.debug(f"Secure environment applied: {msg}") - else: - logger.debug(f"Running in unsecured mode — no certs found. To set up: weightslab ui se") - except Exception as e: - logger.debug(f"Secure environment check skipped: {e}") + success, msg = manager.check_and_apply() + if success: + logger.debug(f"Secure environment applied: {msg}") + else: + logger.debug("Running in unsecured mode - no certs found. To set up: weightslab se") + except Exception as e: + logger.debug(f"Secure environment check skipped: {e}") # Get Package Metadata. Resolve the version from the most authoritative source # available, so a live/editable checkout reports the CURRENT git tag rather than @@ -148,6 +198,9 @@ def _clean(v: str) -> str: "get_samples_by_tag", "get_discarded_samples", "SignalContext", + "BatchSignalContext", + "StaleSignalError", + "drain_signals", "clear_all", "seed_everything", "run_pending_evaluation", @@ -166,6 +219,15 @@ def _clean(v: str) -> str: "write_history", "write_dataframe", + "classify_loss_shape", + "write_loss_shapes", + "write_signal_shapes", + "trajectory_stats", + "enable_loss_shape_signal", + "enable_loss_shape_autotag", + "disable_loss_shape_autotag", + "auto_loss_shape_signal_names", + "LOSS_SHAPES", "pointcloud_thumbnail", "pointcloud_boxes", diff --git a/weightslab/art.py b/weightslab/art.py index 21cdc379..85b62ead 100644 --- a/weightslab/art.py +++ b/weightslab/art.py @@ -96,6 +96,19 @@ def _package_version() -> str: ``_version`` module, then the git-describe fallback. Deliberately avoids ``import weightslab`` because this module is imported *during* the package's own initialization (before ``__version__`` is assigned there).""" + raw = _package_version_raw() + # The git-describe fallback returns the raw tag name (e.g. "v1.3.4" or + # "v1.3.4-2-gabcdef"), since release tags are "v*" — already-prefixed. The + # caller (_CREDIT_LABEL below) always adds its own "v", so strip one here + # if present or the banner reads "vv1.3.4". The other two branches + # (installed dist metadata / setuptools_scm) are unprefixed PEP 440 + # versions, so this is a no-op for them. + if raw[:1] in ("v", "V") and raw[1:2].isdigit(): + raw = raw[1:] + return raw + + +def _package_version_raw() -> str: try: from importlib.metadata import version as _dist_version, PackageNotFoundError try: diff --git a/weightslab/backend/dataloader_interface.py b/weightslab/backend/dataloader_interface.py index d9b16e94..9f76f660 100644 --- a/weightslab/backend/dataloader_interface.py +++ b/weightslab/backend/dataloader_interface.py @@ -1045,6 +1045,26 @@ def reset_iterator(self) -> None: """ self._reset_iterator() + def reset(self) -> None: + """Recreate the internal iterator (Ultralytics / ``InfiniteDataLoader``-compatible). + + Some training frameworks call ``loader.reset()`` to force a fresh + iterator after mutating the dataset mid-run — e.g. Ultralytics closes + mosaic augmentation near the end of training and then calls + ``train_loader.reset()`` so the change takes effect on the next epoch. + We map it to ``_reset_iterator()``. + + Exposing this as a real method (rather than relying on a caller-side + shim like ``if not hasattr(loader, "reset"): loader.reset = ...``) is + important: this loader is handed to callers wrapped in a ledger + ``Proxy`` whose ``__getattr__`` returns ``None`` for genuinely-missing + attributes instead of raising ``AttributeError``. That makes + ``hasattr(proxy, "reset")`` return True and silently resolves + ``proxy.reset`` to ``None``, so any such shim is skipped and + ``reset()`` blows up with ``'NoneType' object is not callable``. + """ + self._reset_iterator() + # ------------------------------------------------------------------------- # Iteration state capture/restore for deterministic resume # ------------------------------------------------------------------------- diff --git a/weightslab/backend/ledgers.py b/weightslab/backend/ledgers.py index 76e77ea7..df75f90d 100644 --- a/weightslab/backend/ledgers.py +++ b/weightslab/backend/ledgers.py @@ -500,7 +500,16 @@ def __getattr__(self, item): except (KeyError, TypeError): pass - return None + # Attribute genuinely absent on the wrapped object. Raise AttributeError + # (as the class docstring promises) rather than returning None: returning + # None makes `hasattr(proxy, x)` report True for missing attributes, which + # breaks capability probes (e.g. `if not hasattr(loader, "reset")`) and + # silently resolves `proxy.missing` to a non-callable None. Callers that + # want a soft default should use `getattr(proxy, x, default)`, which now + # works correctly. + raise AttributeError( + f"'{type(obj).__name__}' object has no attribute {item!r}" + ) def __delattr__(self, name: str) -> None: """Delete an attribute from the proxy or wrapped object safely. @@ -582,7 +591,19 @@ def __len__(self): def __getitem__(self, idx): if self._obj is None: raise TypeError("Proxy target not set") - return self._obj[idx] + value = self._obj[idx] + # Mirror .get(): subscript access on a mapping returns a live key proxy + # for "simple" values, so ``b['test']`` tracks studio edits exactly like + # ``b.get('test')``. Lists/slices and non-mapping containers, plus + # list/dict/callable values, are returned raw (unchanged behavior). + if (hasattr(self._obj, "get") and hasattr(self._obj, "keys") + and not (isinstance(value, (list, dict)) or callable(value))): + vp = Proxy._ValueProxy(self, idx) + # str / torch.device are handed back plain (see _plain_get_value); + # other simple types stay live proxies so studio edits keep tracking. + plain = _plain_get_value(vp._resolve()) + return vp if plain is _KEEP_AS_PROXY else plain + return value def __setitem__(self, key, value): """Support item assignment: proxy[key] = value""" @@ -953,6 +974,24 @@ def list_hyperparams(self) -> List[str]: with self._lock: return list(self._hyperparams.keys()) + def has_serving_config(self) -> bool: + """True if any hyperparameters set holds a non-empty mapping. + + Distinguishes a genuine ``register_hyperparams`` from the empty + ``Proxy({})`` placeholder that ``get_hyperparams`` creates on first + access — so callers can tell whether serving config (TLS, ports, + ``root_log_dir``) was actually wrapped before ``serve()``. Checking + ``list_hyperparams()`` alone is not enough: a bare ``get_hyperparams`` + call already leaves an (empty) placeholder behind. + """ + with self._lock: + entries = list(self._hyperparams.values()) + for entry in entries: + resolved = entry.get() if Proxy.is_proxy(entry) else entry + if isinstance(resolved, dict) and len(resolved) > 0: + return True + return False + def set_hyperparam(self, key_path: str, value: Any, name: str = DEFAULT_NAME) -> None: """Set a nested hyperparameter using dot-separated `key_path`. Example: set_hyperparam(name='exp', key_path='data.train.batch_size', value=128) @@ -1323,6 +1362,14 @@ def get_hyperparams(name: str = DEFAULT_NAME) -> Any: def list_hyperparams() -> List[str]: return GLOBAL_LEDGER.list_hyperparams() +def has_serving_config() -> bool: + """True when serving config (hyperparameters) has been wrapped. + + See :meth:`GrayBoxLedger.has_serving_config`. Used by ``wl.serve`` to guard + against exposing an unconfigured backend. + """ + return GLOBAL_LEDGER.has_serving_config() + def resolve_hp_name() -> str | None: """Resolve a sensible hyperparam set name from the ledger. Checks for 'main', then 'experiment', then falls back to the first registered name. diff --git a/weightslab/backend/logger.py b/weightslab/backend/logger.py index 4e6f95b1..3122be62 100644 --- a/weightslab/backend/logger.py +++ b/weightslab/backend/logger.py @@ -29,9 +29,13 @@ staging appends and flushes take the same lock. """ +import functools import json +import logging +import os import threading import time +from collections import defaultdict import duckdb import pandas as pd @@ -39,6 +43,8 @@ from weightslab.backend.ledgers import get_logger, register_logger, get_checkpoint_manager +logger = logging.getLogger(__name__) + # Column order for each table's staging buffer / bulk insert. _SIGNAL_COLS = [ @@ -55,6 +61,13 @@ # many rows, to bound memory during long runs that never read history. _STAGE_FLUSH_THRESHOLD = 50_000 +# How often the background flush thread wakes up (see LoggerQueue._flush_loop). +def _default_flush_interval_seconds() -> float: + try: + return float(os.environ.get("WL_LOGGER_FLUSH_INTERVAL_SECONDS", "2.0")) + except (TypeError, ValueError): + return 2.0 + class LoggerQueue: def __init__(self, register: bool = True, db_path: str = ":memory:") -> None: @@ -73,6 +86,19 @@ def __init__(self, register: bool = True, db_path: str = ":memory:") -> None: self._eval_mode_tags: list[str] = [] self._eval_accum: dict = {} # {graph_name: [sum, count]} + # Background flush + loss-shape autotag state. Every flag="loss" signal + # is auto-classified by default (see _autotag_loss_shapes); these only + # hold user overrides/opt-outs for specific signals (or all of them). + self._loss_shape_overrides: dict = {} # {signal_name: (tag_name, classifier)} + self._loss_shape_disabled: set = set() + self._loss_shape_all_disabled: bool = False + # {signal_name: _qps_version at the last successful tag pass} — lets + # _autotag_loss_shapes() skip a signal entirely when no new per-sample + # data has been staged for it since last time (see _autotag_loss_shapes). + self._loss_shape_last_version: dict = {} + self._flush_stop = threading.Event() + self._flush_thread: threading.Thread | None = None + # DuckDB connection + write-staging buffers. self._lock = threading.RLock() self._db_path = db_path @@ -81,6 +107,20 @@ def __init__(self, register: bool = True, db_path: str = ":memory:") -> None: self._stage_sample: list = [] self._stage_instance: list = [] self._seq = 0 + + # Per-sample query cache. Many consumers read the SAME (signal, ids) in + # one step (e.g. reactive signals all reading the loss); memoize so N + # identical reads cost 1 scan. Keyed by (signal, ids, [step,] hash, + # version[signal]); staging a row bumps that signal's version to + # invalidate. Step-scoped: _stage_sample_row clears both caches when the + # step advances (keys never recur across steps, so old entries are dead). + # Cache size is env-configurable (WL_QUERY_CACHE_MAXSIZE, default 2048). + _qps_maxsize = int(os.environ.get("WL_QUERY_CACHE_MAXSIZE", "2048")) + self._qps_version: dict = defaultdict(int) + self._qps_cache_step: int = -1 + self._qps_cache = functools.lru_cache(maxsize=_qps_maxsize)(self._query_per_sample_uncached) + self._qps_step_cache = functools.lru_cache(maxsize=_qps_maxsize)(self._query_per_sample_at_step_uncached) + self._ensure_tables() self._restore_runtime_state_from_db() @@ -95,45 +135,249 @@ def __init__(self, register: bool = True, db_path: str = ":memory:") -> None: # Init checkpoint manager for experiment hash retrieval (if available) self.chkpt_manager = get_checkpoint_manager() + # If no explicit db_path was given but a checkpoint manager already + # exists, persist history to an on-disk DuckDB file under its loggers/ + # dir. (The reverse ordering — CM created after the logger — is handled + # by CheckpointManager.__init__ calling set_db_path on the live logger.) + if db_path == ":memory:": + try: + loggers_dir = getattr(self.chkpt_manager, "loggers_dir", None) + if loggers_dir: + self.set_db_path(os.path.join(str(loggers_dir), "loggers.duckdb")) + except Exception: + pass + + self._start_background_flush() + + # ------------------------------------------------------------------ + # Background flush + loss-shape autotag + # ------------------------------------------------------------------ + def _start_background_flush(self) -> None: + """Start the periodic flush/loss-shape thread (off the caller's thread). + + Runs for the life of the process (daemon) so neither the training loop + nor the gRPC servicer has to remember to flush or re-tag loss shapes + themselves — every flag="loss" signal is auto-classified with zero + setup. See _autotag_loss_shapes for the tagging half.""" + if self._flush_thread is not None and self._flush_thread.is_alive(): + return + self._flush_stop.clear() + self._flush_thread = threading.Thread( + target=self._flush_loop, name="WL-Logger-Flush", daemon=True) + self._flush_thread.start() + + def set_loss_shape_override(self, loss_signal: str, tag_name: str | None = None, + classifier=None) -> None: + """Override the tag name / classifier used when auto-tagging *loss_signal*. + + Every ``flag="loss"`` signal is already auto-classified with zero setup + (see :meth:`_autotag_loss_shapes`); call this only to customize one + specific signal (e.g. it isn't a decreasing loss, so the default + classifier is wrong for it). Also re-enables that signal if it had been + disabled via :meth:`disable_loss_shape_autotag`. + """ + with self._lock: + self._loss_shape_overrides[loss_signal] = (tag_name, classifier) + self._loss_shape_disabled.discard(loss_signal) + + def disable_loss_shape_autotag(self, loss_signal: str | None = None) -> None: + """Stop automatic loss-shape tagging for *loss_signal*, or for every + signal (including ones registered later) if *loss_signal* is ``None``. + The background flush itself keeps running either way.""" + with self._lock: + if loss_signal is None: + self._loss_shape_all_disabled = True + else: + self._loss_shape_disabled.add(loss_signal) + self._loss_shape_overrides.pop(loss_signal, None) + + def _autotag_loss_shapes(self) -> None: + """Re-classify every auto-detected ``flag="loss"`` signal that has NEW + per-sample data since its last tag pass. + + Each signal is skipped when its ``_qps_version`` (bumped only by + ``_stage_sample_row`` on an actual new per-sample write, and already + the cache-invalidation key for ``query_per_sample``/ + ``query_signal_history``) hasn't moved since the last time this + signal was classified — no new points means the classification result + (and thus every categorical tag written from it) would come out + identical, so re-running the full read + classify + tag-write cycle + on a bare 2s timer regardless of whether training even produced + anything new was pure waste, and contended ``self._lock`` against + unrelated reads (e.g. ``get_signal_history()`` for + ``GetLatestLoggerData``) the whole time training was paused/idle too. + Failures are per-signal and never propagate — one bad + classifier/signal can't stop the others. + """ + with self._lock: + if self._loss_shape_all_disabled: + return + disabled = set(self._loss_shape_disabled) + overrides = dict(self._loss_shape_overrides) + try: + # Lazy import: weightslab.src imports LoggerQueue at module load + # time, so importing it back here at module scope would cycle. + from weightslab.src import auto_loss_shape_signal_names, write_signal_shapes + except Exception as exc: + logger.debug(f"[LoggerQueue] loss-shape autotag: import failed: {exc}") + return + # Union, not just the auto-detected set: set_loss_shape_override() can + # also opt in a signal that wasn't registered via flag="loss" (e.g. a + # manually wl.save_signals()-logged one under a custom name). + signal_names = set(auto_loss_shape_signal_names()) | set(overrides.keys()) + for signal_name in signal_names: + if signal_name in disabled: + continue + with self._lock: + current_version = self._qps_version[signal_name] + if self._loss_shape_last_version.get(signal_name) == current_version: + continue # no new per-sample data logged since the last pass + tag_name, classifier = overrides.get(signal_name, (None, None)) + try: + write_signal_shapes(signal_name, tag_name=tag_name, classifier=classifier) + self._loss_shape_last_version[signal_name] = current_version + except Exception as exc: + logger.debug( + f"[LoggerQueue] loss-shape autotag failed for {signal_name!r}: {exc}") + + def _flush_loop(self) -> None: + interval = _default_flush_interval_seconds() + while not self._flush_stop.wait(interval): + try: + self.flush_to_disk() + except Exception as exc: + logger.debug(f"[LoggerQueue] background flush failed: {exc}") + self._autotag_loss_shapes() + + def stop_background_flush(self) -> None: + """Stop the background flush/loss-shape thread (e.g. at shutdown or in tests).""" + self._flush_stop.set() + if self._flush_thread is not None: + self._flush_thread.join(timeout=2.0) + # ------------------------------------------------------------------ # DuckDB plumbing # ------------------------------------------------------------------ + @staticmethod + def _schema_ddl(prefix: str = "") -> str: + """Return the CREATE-TABLE DDL for all history tables. + + ``prefix`` lets the same schema be created inside an attached database + (e.g. ``"ondisk."``) when migrating from in-memory to a file. + """ + return f""" + CREATE TABLE IF NOT EXISTS {prefix}signals ( + metric_name VARCHAR, + experiment_hash VARCHAR, + step INTEGER, + metric_value DOUBLE, + timestamp BIGINT, + audit_mode BOOLEAN, + is_evaluation_marker BOOLEAN, + split_name VARCHAR, + evaluation_tags VARCHAR, + point_note VARCHAR, + seq BIGINT + ); + CREATE TABLE IF NOT EXISTS {prefix}per_sample ( + metric_name VARCHAR, + experiment_hash VARCHAR, + sample_id VARCHAR, + step INTEGER, + value REAL, + seq BIGINT + ); + CREATE TABLE IF NOT EXISTS {prefix}per_instance ( + metric_name VARCHAR, + experiment_hash VARCHAR, + sample_id VARCHAR, + annotation_id INTEGER, + step INTEGER, + value REAL, + seq BIGINT + ); + """ + def _ensure_tables(self) -> None: with self._lock: - self._conn.execute( - """ - CREATE TABLE IF NOT EXISTS signals ( - metric_name VARCHAR, - experiment_hash VARCHAR, - step INTEGER, - metric_value DOUBLE, - timestamp BIGINT, - audit_mode BOOLEAN, - is_evaluation_marker BOOLEAN, - split_name VARCHAR, - evaluation_tags VARCHAR, - point_note VARCHAR, - seq BIGINT - ); - CREATE TABLE IF NOT EXISTS per_sample ( - metric_name VARCHAR, - experiment_hash VARCHAR, - sample_id VARCHAR, - step INTEGER, - value REAL, - seq BIGINT - ); - CREATE TABLE IF NOT EXISTS per_instance ( - metric_name VARCHAR, - experiment_hash VARCHAR, - sample_id VARCHAR, - annotation_id INTEGER, - step INTEGER, - value REAL, - seq BIGINT - ); - """ - ) + self._conn.execute(self._schema_ddl()) + + _HISTORY_TABLES = ("signals", "per_sample", "per_instance") + + def set_db_path(self, db_path) -> None: + """Persist signal history to an on-disk DuckDB file. + + Call this once, early in setup. If the file already exists (resume), + its data is adopted as-is. If it does not, whatever little is currently + in the in-memory DB is migrated into the new file. Either way the file + becomes the live connection afterwards. + + The hot logging path is unaffected: ``add_scalars`` still stages to RAM + and only bulk-flushes to DuckDB lazily. DuckDB serves reads from its + in-memory buffer pool, so this adds durability, not per-read disk hits. + """ + if not db_path or db_path == ":memory:": + return + db_path = str(db_path) + + with self._lock: + if self._db_path == db_path: + return + + parent = os.path.dirname(db_path) + if parent: + os.makedirs(parent, exist_ok=True) + + file_preexists = os.path.exists(db_path) + + try: + # Make sure staged rows are materialized before we migrate. + self._flush_stage() + + if not file_preexists: + # Fresh file: copy whatever is in the in-memory DB into it. + # ATTACH doesn't accept bind parameters, so inline the path + # with SQL-escaped quotes. + escaped = db_path.replace("'", "''") + self._conn.execute(f"ATTACH '{escaped}' AS ondisk") + self._conn.execute(self._schema_ddl(prefix="ondisk.")) + for tbl in self._HISTORY_TABLES: + self._conn.execute( + f"INSERT INTO ondisk.{tbl} SELECT * FROM {tbl}" + ) + self._conn.execute("DETACH ondisk") + + # Adopt the on-disk file as the live connection. On resume this + # is the source of truth; the fresh in-memory rows are ignored. + self._conn.close() + self._conn = duckdb.connect(database=db_path) + self._db_path = db_path + self._ensure_tables() + self._invalidate_qps_cache() + self._restore_runtime_state_from_db() + logger.info( + f"[LoggerQueue] Signal history persisted on disk at {db_path} " + f"({'adopted existing' if file_preexists else 'new'} database)." + ) + except Exception as exc: + logger.warning( + f"[LoggerQueue] Failed to enable on-disk persistence at " + f"{db_path}: {exc}. Keeping in-memory history." + ) + + def flush_to_disk(self) -> None: + """Flush staged rows and force a DuckDB checkpoint to the file. + + No-op for an in-memory database. Call at checkpoint time so history is + durable even without a clean shutdown (DuckDB also replays its WAL on + the next open, so this is belt-and-braces).""" + with self._lock: + try: + self._flush_stage() + if self._db_path != ":memory:": + self._conn.execute("CHECKPOINT") + except Exception as exc: + logger.warning(f"[LoggerQueue] flush_to_disk failed: {exc}") def _restore_runtime_state_from_db(self) -> None: """Repopulate seq counter and graph names from an existing (file) DB.""" @@ -167,7 +411,10 @@ def _maybe_autoflush(self) -> None: self._flush_stage() def _flush_stage(self) -> None: - """Bulk-insert all staged rows into DuckDB and clear the buffers.""" + """Bulk-insert all staged rows into DuckDB and clear the buffers. + + Uses register(pandas)->INSERT SELECT->unregister (DuckDB's fast bulk + path). A row-wise executemany was measured ~6x slower — don't switch.""" with self._lock: if self._stage_signals: df = pd.DataFrame(self._stage_signals, columns=_SIGNAL_COLS) @@ -198,11 +445,25 @@ def _stage_signal_row(self, graph_name, exp_hash, step, metric_value, timestamp, self._maybe_autoflush() def _stage_sample_row(self, graph_name, exp_hash, sample_id, step, value): - self._stage_sample.append(( - graph_name, exp_hash, str(sample_id), int(step), float(value), self._next_seq(), - )) + # New step -> last step's cache entries can't recur; drop them. + if int(step) > self._qps_cache_step: + self._invalidate_qps_cache() + self._qps_cache_step = int(step) + self._stage_sample.append( + ( + graph_name, exp_hash, str(sample_id), int(step), float(value), self._next_seq(), + ) + ) + self._qps_version[graph_name] += 1 # invalidate this signal's cached reads self._maybe_autoflush() + def _invalidate_qps_cache(self) -> None: + """Drop both query caches + versions (step advance; bulk delete/clear).""" + self._qps_cache.cache_clear() + self._qps_step_cache.cache_clear() + self._qps_version.clear() + self._loss_shape_last_version.clear() + def _stage_instance_row(self, graph_name, exp_hash, sample_id, annotation_id, step, value): self._stage_instance.append(( graph_name, exp_hash, str(sample_id), int(annotation_id), int(step), @@ -244,6 +505,7 @@ def clear_signal_histories(self): self._conn.execute("DELETE FROM per_instance") self._current_step_buffer.clear() self._buffered_step = None + self._invalidate_qps_cache() def _to_float(self, value): if isinstance(value, th.Tensor): @@ -443,6 +705,7 @@ def remove_evaluation_hash(self, eval_hash: str) -> None: self._flush_stage() self._conn.execute("DELETE FROM signals WHERE experiment_hash = ?", [eval_hash]) self._conn.execute("DELETE FROM per_sample WHERE experiment_hash = ?", [eval_hash]) + self._invalidate_qps_cache() # Drop queued points that reference this hash. self._pending_queue = [ @@ -712,19 +975,75 @@ def query_per_sample(self, graph_name: str, sample_ids=None, exp_hash=None): Returns a list of ``(sample_id, step, value, experiment_hash)`` tuples, filtered by *sample_ids* and optionally *exp_hash* (``None`` = all hashes). + + Cached (memoized until *graph_name* is next staged). Returns a fresh list. """ + ids_key = tuple(str(s) for s in sample_ids) if sample_ids is not None else None + cached = self._qps_cache(graph_name, ids_key, exp_hash, self._qps_version[graph_name]) + return list(cached) + + def _query_per_sample_uncached(self, graph_name, ids_key, exp_hash, _version): + """DuckDB read behind :meth:`query_per_sample`. ``_version`` is a cache + key only (bumped on write -> recompute). Returns an immutable tuple.""" with self._lock: self._flush_stage() params = [graph_name] sql = "SELECT sample_id, step, value, experiment_hash FROM per_sample WHERE metric_name = ?" sql += self._hash_filter(exp_hash, params) - if sample_ids is not None: + if ids_key is not None: sql += " AND sample_id IN (SELECT UNNEST(?))" - params.append([str(s) for s in sample_ids]) + params.append(list(ids_key)) sql += " ORDER BY seq" rows = self._conn.execute(sql, params).fetchall() - return [(sid, int(step), float(val), h) for (sid, step, val, h) in rows] + return tuple((sid, int(step), float(val), h) for (sid, step, val, h) in rows) + + def query_per_sample_at_step(self, graph_name: str, sample_ids, step, exp_hash=None): + """``(sample_id, value)`` for *graph_name* at exactly *step* — O(batch), + not O(history). Keeps the reactive gather flat as history grows. Cached.""" + ids_key = tuple(str(s) for s in sample_ids) if sample_ids is not None else None + cached = self._qps_step_cache(graph_name, ids_key, int(step), exp_hash, + self._qps_version[graph_name]) + return list(cached) + + def _query_per_sample_at_step_uncached(self, graph_name, ids_key, step, exp_hash, _version): + """DuckDB read behind :meth:`query_per_sample_at_step` (``_version`` = cache key). + + Fast path: the current step's value is usually still in the in-memory + staging buffer, so scan it and skip the flush->register->INSERT->SELECT + round-trip. Fall through to DuckDB only if an id isn't staged.""" + step = int(step) + with self._lock: + if ids_key is not None: + ids_set = set(ids_key) + at = {} + # Scan from the tail (append-ordered by step); first value per id + # wins, stop when all found or once we drop below `step`. + for row in reversed(self._stage_sample): + s = row[3] + if s < step: + break + if s == step and row[0] == graph_name \ + and (exp_hash is None or row[1] == exp_hash): + sid = row[2] + if sid in ids_set and sid not in at: + at[sid] = row[4] + if len(at) == len(ids_set): + break + if len(at) == len(ids_set): + return tuple((sid, float(val)) for sid, val in at.items()) + + # Fallback: not fully in the staging buffer -> flush + query DuckDB. + self._flush_stage() + params = [graph_name, step] + sql = "SELECT sample_id, value FROM per_sample WHERE metric_name = ? AND step = ?" + sql += self._hash_filter(exp_hash, params) + if ids_key is not None: + sql += " AND sample_id IN (SELECT UNNEST(?))" + params.append(list(ids_key)) + rows = self._conn.execute(sql, params).fetchall() + + return tuple((sid, float(val)) for (sid, val) in rows) def query_per_instance( self, diff --git a/weightslab/backend/model_interface.py b/weightslab/backend/model_interface.py index 6f6fc367..a3f02bc2 100755 --- a/weightslab/backend/model_interface.py +++ b/weightslab/backend/model_interface.py @@ -314,6 +314,32 @@ def audit_mode(self) -> bool: return False + @staticmethod + def _active_model_in_audit_mode() -> bool: + """Return True iff the *currently registered* experiment model is in audit mode. + + The backward/optimizer overrides below are installed on the torch classes + exactly once, process-wide. They must therefore consult whichever model is + active *now* (resolved from the ledger, the same way GuardContext does) — + NOT the ModelInterface instance that happened to install the patch first. + Capturing a single instance in the closure is a bug: a transient eval-mode + model (e.g. one built only for dependency extraction) would then make every + backward()/step() in the whole process a silent no-op. + """ + try: + model = ledgers.get_model() + except Exception: + return False + # get_model() may hand back a Proxy(None) placeholder when nothing is + # registered; unwrap it and treat "no model" as "not in audit mode". + try: + model = object.__getattribute__(model, '_obj') + except AttributeError: + pass + if model is None: + return False + return bool(getattr(model, 'audit_mode', False)) + def _setup_backward_override(self): """Set up monkey-patch to disable backward() when model is in audit/eval mode. @@ -323,12 +349,11 @@ def _setup_backward_override(self): """ # Store the original backward method original_backward = th.Tensor.backward - model_interface_ref = self def backward_override(tensor_self, gradient=None, retain_graph=False, create_graph=False, inputs=None): """Override backward to be a no-op in audit mode.""" - # Check if model is in audit mode via the audit_mode property - if model_interface_ref.audit_mode: + # Resolve the active model at call time (never a stale captured ref). + if ModelInterface._active_model_in_audit_mode(): return # Otherwise, call the original backward @@ -355,12 +380,11 @@ def _setup_optimizer_step_override(self): """ # Store the original step method original_step = th.optim.Optimizer.step - model_interface_ref = self def step_override(self, closure=None): """Override step to be a no-op in audit mode.""" - # Check if the model is in audit mode via the audit_mode property - if model_interface_ref.audit_mode: + # Resolve the active model at call time (never a stale captured ref). + if ModelInterface._active_model_in_audit_mode(): return # Otherwise, call the original step diff --git a/weightslab/cli.py b/weightslab/cli.py new file mode 100644 index 00000000..63c55076 --- /dev/null +++ b/weightslab/cli.py @@ -0,0 +1,838 @@ +"""WeightsLab command-line interface. + +Docker-free. Commands: + * ``weightslab start`` — run the native UI server (bundled SPA + + gRPC-Web proxy), the pip-native replacement + for the old Docker/Envoy stack. + * ``weightslab start example`` — run a bundled PyTorch example. + * ``weightslab se`` — set up the secure environment (TLS certs + + gRPC auth token) in $WEIGHTSLAB_CERTS_DIR. + * ``weightslab cli`` — attach a terminal to a running experiment. + * ``weightslab tunnel`` — forward a remote gRPC backend to a local port. +""" + +import argparse +import os +import stat +import subprocess +import sys +import logging +from pathlib import Path +from typing import Optional, Sequence + +import yaml + +from weightslab.security import CertAuthManager +from weightslab.tunnel import DEFAULT_LISTEN_PORT + +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +logger = logging.getLogger(__name__) + +# Capture whether WEIGHTSLAB_CERTS_DIR was already in the shell env when this +# process started (set by the user) vs. injected later by our own code. +_CERTS_DIR_IN_ORIGINAL_ENV: bool = "WEIGHTSLAB_CERTS_DIR" in os.environ + + +def _coerce_valid_port(value) -> Optional[int]: + """Convert a value to a valid TCP port, or None when invalid.""" + try: + port = int(value) + except (TypeError, ValueError): + return None + return port if 1 <= port <= 65535 else None + + +def _lookup_nested(mapping: dict, path: Sequence[str]): + """Safely resolve a nested dict path and return None on missing keys.""" + cur = mapping + for key in path: + if not isinstance(cur, dict) or key not in cur: + return None + cur = cur[key] + return cur + + +def _load_ui_port_from_experiment_config(explicit_path: Optional[str] = None) -> Optional[int]: + """Read a preferred UI port from an experiment config file when available. + + Resolution order: + 1) explicit_path (if provided) + 2) $WEIGHTSLAB_EXPERIMENT_CONFIG + 3) ./config.yaml, ./config.yml, ./experiment_config.yaml, ./experiment_config.yml + """ + candidates = [] + if explicit_path: + candidates.append(Path(explicit_path)) + env_cfg = os.getenv("WEIGHTSLAB_EXPERIMENT_CONFIG", "").strip() + if env_cfg: + candidates.append(Path(env_cfg)) + cwd = Path.cwd() + candidates.extend([ + cwd / "config.yaml", + cwd / "config.yml", + cwd / "experiment_config.yaml", + cwd / "experiment_config.yml", + ]) + + first_existing = next((p for p in candidates if p.exists() and p.is_file()), None) + if first_existing is None: + return None + + try: + payload = yaml.safe_load(first_existing.read_text(encoding="utf-8")) + except Exception as exc: + logger.debug(f"Could not parse experiment config {first_existing}: {exc}") + return None + + if not isinstance(payload, dict): + return None + + candidate_values = [ + payload.get("ui_port"), + payload.get("studio_port"), + _lookup_nested(payload, ("studio", "port")), + _lookup_nested(payload, ("weightslab", "ui_port")), + _lookup_nested(payload, ("weightslab", "studio", "port")), + ] + for val in candidate_values: + coerced = _coerce_valid_port(val) + if coerced is not None: + return coerced + return None + + +def _resolve_ui_port(args) -> tuple[int, str]: + """Resolve preferred UI port source: arg > config > env > default.""" + arg_port = _coerce_valid_port(getattr(args, "port", None)) + if arg_port is not None: + return arg_port, "--port" + + config_port = _load_ui_port_from_experiment_config(getattr(args, "config", None)) + if config_port is not None: + return config_port, "experiment config" + + env_port = _coerce_valid_port(os.getenv("WL_LAST_UI_PORT")) + if env_port is not None: + return env_port, "WL_LAST_UI_PORT" + + compat_env_port = _coerce_valid_port(os.getenv("WEIGHTSLAB_UI_PORT")) + if compat_env_port is not None: + return compat_env_port, "WEIGHTSLAB_UI_PORT" + + return 50051, "default" + + +def _persist_certs_dir(certs_dir_str: str) -> None: + """Persist WEIGHTSLAB_CERTS_DIR so future terminals and the training backend find it. + + Windows — runs `setx` (permanent user env) and prints the PS one-liner for + the current session. + Linux/macOS — appends an export line to ~/.bashrc (idempotent) and prints the + source command for the current session. + """ + export_line = f'export WEIGHTSLAB_CERTS_DIR="{certs_dir_str}"' + if _is_windows(): + result = subprocess.run( + ["setx", "WEIGHTSLAB_CERTS_DIR", certs_dir_str], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + if result.returncode == 0: + logger.info(" WEIGHTSLAB_CERTS_DIR saved permanently via setx (new terminals will have it)") + else: + logger.warning(f"setx failed — set it manually: setx WEIGHTSLAB_CERTS_DIR \"{certs_dir_str}\"") + logger.info(f" Current terminal (PowerShell): $env:WEIGHTSLAB_CERTS_DIR = \"{certs_dir_str}\"") + else: + bashrc = Path.home() / ".bashrc" + try: + existing = bashrc.read_text(encoding="utf-8") if bashrc.exists() else "" + if export_line not in existing: + with open(bashrc, "a", encoding="utf-8") as f: + f.write(f"\n# Added by weightslab\n{export_line}\n") + logger.info(f" WEIGHTSLAB_CERTS_DIR appended to {bashrc} (new terminals will have it)") + else: + logger.info(f" WEIGHTSLAB_CERTS_DIR already in {bashrc}") + except OSError as e: + logger.warning(f"Could not write to {bashrc}: {e}") + logger.info(f" Add manually: {export_line}") + logger.info(f" Current terminal: source ~/.bashrc (or open a new terminal)") + + +def _banner() -> str: + """Return the WeightsLab ASCII banner, or a plain title if art is unavailable.""" + try: + from weightslab.art import _BANNER + return _BANNER + except Exception: + return "WeightsLab" + + +_DESCRIPTION = ( + _banner() + + "\nWeightsLab — Inspect, Edit, and Evolve Neural Networks\n" + + "Run the Weights Studio UI, bundled examples, and the secure " + + "(TLS + gRPC auth) environment." +) + +_EPILOG = """\ +commands: + se Set up the secure environment: generate TLS + certificates + a gRPC auth token in + ~/.weightslab-certs. Then set WEIGHTSLAB_CERTS_DIR + (the single source of truth) so the backend + new + shells find them. + --force-certs regenerate even if certs exist + + start Start the Weights Studio UI natively — no Docker. + Serves the bundled SPA and proxies gRPC-Web to a + running backend, all from one Python process. + UNSECURED (HTTP) by default. + --port PORT UI HTTP port (default 50051) + --config FILE experiment config file used to read ui_port + --backend-port PORT backend gRPC port (default 50051) + --backend-host HOST backend gRPC host (default localhost) + --no-browser don't open a browser + --certs HTTPS + mTLS from $WEIGHTSLAB_CERTS_DIR + + start example Run a bundled PyTorch example (foreground; stop with + Ctrl+C). Installs the example's requirements first, + without prompting. Defaults to classification: + --cls classification example (default) + --seg segmentation example + --det detection example + --clus clustering example + --gen generation example + --3d_det 3D LiDAR point-cloud detection example + --2d_det 2D LiDAR point-cloud detection example + + cli Open an interactive terminal connected to a + currently-running experiment (pause/resume, status, + evaluate, agent query, etc.). Auto-discovers the + running experiment; the experiment must be serving + the CLI (e.g. wl.serve(serving_cli=True)). + --port PORT connect to a specific CLI port + --host HOST connect to a specific host (default: localhost) + + tunnel Forward a REMOTE gRPC backend (e.g. a Colab run + behind `ngrok tcp 50051`) to a LOCAL port so + `weightslab start` can proxy to it. Raw TCP, so the + backend must be plaintext (the default). + ENDPOINT remote host:port (e.g. bore.pub:12345) + (default: $WEIGHTSLAB_TUNNEL_ENDPOINT) + --listen-port N local port to expose (default 50051) + --listen-host H interface to bind (default: auto) + --remote-port N remote port, if not in ENDPOINT + +examples: + weightslab se # one-time secure setup (then export WEIGHTSLAB_CERTS_DIR) + weightslab se --force-certs # regenerate the certs + weightslab start # launch the UI (unsecured HTTP, default) at :50051 + weightslab start --certs # launch the UI over HTTPS (needs `weightslab se` first) + weightslab start --port 9000 # launch the UI on a custom port + weightslab start --backend-port 50052 # proxy to a backend on a custom gRPC port + weightslab start example # run the classification demo (default) + weightslab start example --seg # run the segmentation demo + weightslab start example --det # run the detection demo + weightslab start example --3d_det # run the 3D LiDAR detection demo + weightslab cli # connect a terminal to the running experiment + weightslab cli --port 60000 # connect to a specific CLI port + weightslab tunnel bore.pub:12345 # expose a remote (Colab) backend at localhost:50051 +""" + + +def _get_cert_script() -> Path: + """Get the generate-certs-auth-token.sh script path.""" + return Path(__file__).parent / 'ui' / 'utils' / 'generate-certs-auth-token.sh' + + +def _get_cert_script_ps1() -> Path: + """Get the generate-certs-auth-token.ps1 script path.""" + return Path(__file__).parent / 'ui' / 'utils' / 'generate-certs-auth-token.ps1' + + +def _is_windows() -> bool: + """Check if running on Windows.""" + return sys.platform == 'win32' + + +def _make_executable(path) -> None: + """Add the execute bit to a file so it can be run directly (POSIX only). + + pip installs the bundled ``.sh`` scripts as package data *without* the execute + bit, and they are committed/shipped non-executable. ``_run_shell_script`` runs + them as a command (``bash -c "VAR=... '/path/script.sh'"``), which requires + the execute bit — otherwise bash fails with "Permission denied" until the user + ``chmod +x`` by hand. This grants u+x,g+x,o+x (e.g. 0644 -> 0755) and is a + best-effort no-op on Windows, where the POSIX execute bit is not used. + """ + if _is_windows(): + return + try: + mode = os.stat(path).st_mode + os.chmod(path, mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + except OSError as exc: + logger.debug(f"Could not set execute bit on {path}: {exc}") + + +def _ensure_scripts_executable() -> None: + """Make the bundled cert-generation shell script executable (POSIX only). + + Covers generate-certs-auth-token.sh under ``weightslab/ui/utils`` so a freshly + pip-installed package can run it without the user having to ``chmod +x`` first. + No-op on Windows. + """ + if _is_windows(): + return + utils_dir = Path(__file__).parent / 'ui' / 'utils' + try: + scripts = list(utils_dir.rglob('*.sh')) + except OSError as exc: + logger.debug(f"Could not enumerate bundled scripts under {utils_dir}: {exc}") + return + for script in scripts: + _make_executable(script) + + +def _run_powershell_script(script_path: str, args: list = None, env_vars: dict = None) -> int: + """Run a PowerShell script and return exit code.""" + if not _is_windows(): + logger.error("PowerShell certificate generation requires Windows") + return 1 + + cmd = [ + 'powershell', + '-NoProfile', + '-ExecutionPolicy', 'Bypass', + '-File', script_path + ] + + if args: + cmd.extend(args) + + try: + env = os.environ.copy() + if env_vars: + env.update(env_vars) + result = subprocess.run(cmd, env=env) + return result.returncode + except Exception as e: + logger.error(f"Failed to run script: {e}") + return 1 + + +def _convert_to_git_bash_path(win_path: str) -> str: + """Convert Windows path to Git Bash compatible format.""" + # Normalize backslashes to forward slashes explicitly: Path(...).as_posix() + # does NOT convert '\' on POSIX hosts (e.g. Linux CI runners), so a Windows + # input path would keep its separators there. + p = str(win_path).replace("\\", "/") + # Convert C:/Users/... to /mnt/c/Users/... for Git Bash + if len(p) > 1 and p[1] == ':': + drive = p[0].lower() + rest = p[2:] + return f"/mnt/{drive}{rest}" + return p + + +def _run_shell_script(script_path: str, args: list = None, env_vars: dict = None) -> int: + """Run a shell script using bash with proper environment variable passing.""" + + try: + # Fix line endings in the file before running + with open(script_path, 'rb') as f: + script_bytes = f.read() + + # Ensure Unix line endings + fixed_bytes = script_bytes.replace(b'\r\n', b'\n').replace(b'\r', b'\n') + + # Write back if needed + if fixed_bytes != script_bytes: + with open(script_path, 'wb') as f: + f.write(fixed_bytes) + + # We invoke the script as a command below (bash -c "VAR=... 'script'"), + # which needs the execute bit. pip installs it without one, so add it here + # (no-op on Windows). Uses the on-disk path before any Git Bash conversion. + _make_executable(script_path) + + env = os.environ.copy() + if env_vars: + env.update(env_vars) + # Debug: log all environment variables being passed + for key, value in env_vars.items(): + logger.info(f"Passing env var: {key}={value}") + + # Build bash command - pass Windows path directly, script will handle conversion + # # Process path to ensure it's compatible with bash, especially on Windows + if _is_windows() and '\\' in script_path: + script_path = script_path.replace("\\", "/") # Ensure path is Unix-style for bash + script_path = _convert_to_git_bash_path(script_path) + logger.info(f"Converted script path for bash: {script_path}") + logger.info(f"Running shell script: {script_path} with args: {args} and env_vars: {env_vars}") + + # Build environment variable assignments for bash command + env_assignments = ' '.join([f"{k}='{v}'" for k, v in env_vars.items()]) if env_vars else "" + + if env_assignments: + # Pass env vars directly in bash command using -c flag + # This works on Windows (Git Bash), Linux, and macOS + logger.info('Using env assignments') + bash_script_cmd = f"{env_assignments} '{script_path}'" + if args: + bash_script_cmd += " " + " ".join(f"'{arg}'" for arg in args) + bash_cmd = ['bash', '-c', bash_script_cmd] + else: + logger.info('Not using env assignments') + bash_cmd = ['bash', '-x', str(script_path)] + if args: + bash_cmd.extend(args) + + result = subprocess.run(bash_cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if result.stdout: + for line in result.stdout.splitlines(): + logger.info(line) + return result.returncode + except FileNotFoundError: + logger.error(f"Script file not found: {script_path}") + return 1 + except Exception as e: + logger.error(f"Failed to run script: {e}") + return 1 + + +def _generate_certs_with_fallback(force_certs: bool = False, certs_dir=None) -> int: + """Try shell script first, fall back to PowerShell on Windows if it fails. + + ``certs_dir`` is forwarded to the generation scripts as ``WEIGHTSLAB_CERTS_DIR`` + so certs land in the single source-of-truth directory (the scripts default to + ``~/.weightslab-certs`` when it is not provided). The shell script receives a + POSIX absolute path (``/mnt/c/...`` on Windows/WSL); PowerShell receives the + host-native path (``C:/...``). + """ + env_vars = None + if certs_dir is not None: + # Shell scripts (bash/WSL) need a POSIX-style absolute path. + if _is_windows(): + bash_certs_dir = _convert_to_git_bash_path(str(certs_dir)) + else: + bash_certs_dir = str(certs_dir) + env_vars = {'WEIGHTSLAB_CERTS_DIR': bash_certs_dir} + + cert_script = str(_get_cert_script()) + if not Path(cert_script).exists(): + logger.warning(f"Shell script not found: {cert_script}") + else: + script_args = [] + if force_certs: + script_args.append('--force-create-certs') + + logger.info("Attempting certificate generation with shell script...") + exit_code = _run_shell_script(cert_script, script_args, env_vars) + if exit_code == 0: + return 0 + logger.warning(f"Shell script failed (exit code {exit_code})") + + # Fallback to PowerShell on Windows + if _is_windows(): + logger.info("Falling back to PowerShell for certificate generation...") + cert_script_ps1 = str(_get_cert_script_ps1()) + if not Path(cert_script_ps1).exists(): + logger.error(f"PowerShell script not found: {cert_script_ps1}") + return 1 + + script_args = [] + if force_certs: + script_args.append('-ForceCreateCerts') + + exit_code = _run_powershell_script(cert_script_ps1, script_args, env_vars) + return exit_code + else: + logger.error("Neither shell nor PowerShell script could generate certificates") + return 1 + + +def ui_secure_environment(args): + """`weightslab se`: create a certs directory with certs + gRPC token. + + The directory is the single source of truth — WEIGHTSLAB_CERTS_DIR is exported + for this process and the user is asked to export it globally. Everything else + (TLS on/off, auth on/off) is derived from the files in that directory by the + backend and `weightslab start --certs`, so this command sets no other env. + """ + logger.info("Setting up secure environment...") + # Bundled .sh scripts ship without the execute bit (pip strips it); make them + # runnable so the cert-generation script below doesn't fail on "Permission denied". + _ensure_scripts_executable() + + force_certs = getattr(args, "force_certs", False) + no_auth = getattr(args, "no_auth", False) + certs_dir = getattr(args, "certs_dir", None) + if certs_dir: + # Absolute path so Windows Python, WSL bash and the server agree on location. + certs_dir = str(Path(certs_dir).resolve()) + + # Resolve the target directory first (no filesystem work in __init__), so we + # can point the generation scripts at it via WEIGHTSLAB_CERTS_DIR. + manager = CertAuthManager(certs_dir=certs_dir, enable_auth=not no_auth) + + exit_code = _generate_certs_with_fallback(force_certs=force_certs, certs_dir=manager.certs_dir) + if exit_code != 0: + logger.error("Certificate generation failed") + sys.exit(1) + + manager.certs_dir.mkdir(parents=True, exist_ok=True) + manager.get_or_create_auth_token() + + # Export ONLY the single source of truth for this process. + os.environ["WEIGHTSLAB_CERTS_DIR"] = str(manager.certs_dir) + + logger.info(" Certificates generated successfully") + logger.info(" gRPC auth token created") + logger.info(f" Certs and token stored in: {manager.certs_dir}") + logger.info(f" WEIGHTSLAB_CERTS_DIR exported for this process: {manager.certs_dir}") + logger.info("Then launch the secured UI with: weightslab start --certs") + # Keep this the FINAL output so the user can't miss the action they must take. + logger.warning("") + logger.warning(" ACTION REQUIRED — set WEIGHTSLAB_CERTS_DIR globally so new shells " + "and the training backend find these certs (single source of truth):") + logger.warning(f" (bash) echo 'export WEIGHTSLAB_CERTS_DIR=\"{manager.certs_dir}\"' >> ~/.bashrc && source ~/.bashrc") + logger.warning(f" (Windows) setx WEIGHTSLAB_CERTS_DIR \"{manager.certs_dir}\"") + + +# Bundled PyTorch examples, keyed by the CLI flag (e.g. --cls -> wl-classification). +# kind -> (dir_name, label, category) where category is the examples/ subfolder. +_EXAMPLES = { + "cls": ("wl-classification", "classification", "PyTorch"), + "seg": ("wl-segmentation", "segmentation", "PyTorch"), + "det": ("wl-detection", "detection", "PyTorch"), + "clus": ("wl-clustering", "clustering", "PyTorch"), + "gen": ("wl-generation", "generation", "PyTorch"), + "3d_det": ("wl-3d-lidar-detection", "3D LiDAR detection", "Usecases"), + "2d_det": ("wl-2d-lidar-detection", "2D LiDAR detection", "Usecases"), +} +_DEFAULT_EXAMPLE = "cls" + + +def _get_example_dir(name: str = "wl-classification", category: str = "PyTorch") -> Path: + """Path to a bundled example directory (under examples//).""" + return Path(__file__).parent / 'examples' / category / name + + +def _install_example_requirements(example_dir: Path) -> None: + """Install an example's requirements non-interactively, if a file is present. + + Looks for requirements.txt (then requirements.in) in the example directory and + runs `pip install -r` with the current interpreter and `--no-input` so it never + prompts. Non-fatal: a failure is logged and the example is still attempted, so a + transient install hiccup doesn't block a run where deps are already satisfied. + """ + for fname in ("requirements.txt", "requirements.in"): + req = example_dir / fname + if not req.exists(): + continue + logger.info(f"Installing example requirements (non-interactive): {req}") + try: + subprocess.run( + [sys.executable, "-m", "pip", "install", "-r", str(req), + "--no-input", "--disable-pip-version-check"], + check=True, + ) + except subprocess.CalledProcessError as exc: + logger.warning( + f"Failed to install requirements ({req}): {exc}. " + "Continuing — the example may still run if deps are already installed." + ) + return # only the first matching requirements file is used + + +def example_start(args): + """`weightslab start example [--cls|--seg|--clus|--gen]`: run a bundled example. + + Defaults to the classification (cls) example. First installs the example's + requirements (if a requirements file is present) without prompting, then runs + its main.py with the current Python interpreter from its own directory so it + resolves its sibling config.yaml. Runs in the foreground (serves until Ctrl+C). + """ + kind = getattr(args, "example_kind", None) or _DEFAULT_EXAMPLE + dir_name, label, category = _EXAMPLES.get(kind, _EXAMPLES[_DEFAULT_EXAMPLE]) + + example_dir = _get_example_dir(dir_name, category) + main_py = example_dir / "main.py" + if not main_py.exists(): + logger.error(f"{label.capitalize()} example not found: {main_py}") + sys.exit(1) + + # Install the example's own requirements first, without any interaction. + _install_example_requirements(example_dir) + + logger.info(f"Starting the WeightsLab {label} ({kind}) example...") + logger.info(f" {main_py}") + logger.info("In another terminal, launch the UI with: weightslab start") + logger.info("Then open the URL printed by `weightslab start` — stop the example with Ctrl+C.") + if not _CERTS_DIR_IN_ORIGINAL_ENV: + manager = CertAuthManager.from_env_or_default() + if manager.has_valid_certs(): + logger.warning( + "WEIGHTSLAB_CERTS_DIR is not set in your shell environment. " + "TLS will work this session (certs found at default location) " + "but may not persist across terminals." + ) + _persist_certs_dir(str(manager.certs_dir)) + try: + env = os.environ.copy() + env['WEIGHTSLAB_SUPPRESS_BANNER'] = '1' + result = subprocess.run([sys.executable, str(main_py)], cwd=str(example_dir), env=env) + except KeyboardInterrupt: + logger.info("Example stopped.") + return + if result.returncode != 0: + sys.exit(result.returncode) + + +def cli_connect(args): + """`weightslab cli [--port N] [--host H]`: open an interactive terminal + connected to a currently-running experiment's CLI server. + + With no --port, auto-discovers the running experiment (the backend advertises + its actual port on startup). Pass --port to target a specific server. + """ + try: + import weightslab.backend.cli as cli_backend + except Exception as exc: + logger.error(f"Could not load the WeightsLab CLI client: {exc}") + sys.exit(1) + + port = getattr(args, "port", None) + host = getattr(args, "host", None) + exit_code = cli_backend.cli_connect(cli_port=port, cli_host=host) + sys.exit(exit_code) + + +def ui_start_native(args): + """`weightslab start`: launch the Weights Studio UI natively (no Docker). + + Serves the pre-built SPA vendored in this package and proxies gRPC-Web to a + running backend (started by ``wl.serve()``), all from one pure-Python HTTP + server — like ``tensorboard``, the UI ships in the wheel. + + Unsecured HTTP by default. Pass ``--certs`` to serve HTTPS + mTLS to the + backend, derived solely from cert-file presence in $WEIGHTSLAB_CERTS_DIR. + """ + try: + from weightslab.ui import server as ui_server + except Exception as exc: # pragma: no cover - import guard + logger.error(f"Could not load the WeightsLab UI server: {exc}") + sys.exit(1) + + ui_host = getattr(args, "host", None) or os.getenv("WEIGHTSLAB_UI_HOST", "0.0.0.0") + preferred_ui_port, ui_port_source = _resolve_ui_port(args) + ui_port = preferred_ui_port + backend_host = (getattr(args, "backend_host", None) + or os.getenv("GRPC_BACKEND_HOST", "localhost")) + backend_port = (getattr(args, "backend_port", None) + or int(os.getenv("GRPC_BACKEND_PORT", "50051"))) + open_browser = not getattr(args, "no_browser", False) + + # Never bind the UI server on the same TCP port as the backend gRPC target. + # If `weightslab start` grabs backend_port first, the training process cannot + # start its gRPC server and the UI appears disconnected. + if ui_port == backend_port: + logger.warning( + f"Requested UI port {ui_port} matches backend gRPC port {backend_port}; " + "choosing a random free UI port to avoid collision." + ) + ui_port = 0 + + # TLS/auth: single source of truth is cert-file presence in the certs dir. + # Only consulted when the user explicitly opts in with --certs. + certs_dir = None + grpc_auth_token = None + if getattr(args, "certs", False): + manager = CertAuthManager.from_env_or_default() + if manager.has_valid_certs(): + certs_dir = str(manager.certs_dir) + try: + grpc_auth_token = manager.get_or_create_auth_token() + except Exception: + grpc_auth_token = None + else: + logger.warning( + f"--certs requested but no valid certs in {manager.certs_dir}. " + "Run `weightslab se` first. Falling back to unsecured HTTP." + ) + + if not ui_server.has_static_assets(): + logger.warning( + "No bundled UI assets found in this install. The gRPC-Web proxy will " + "still run, but the web page is unavailable. Build the frontend and " + "vendor it into this package (weights_studio: `ui/utils/build-and-deploy.sh`, " + "then `weightslab/ui/utils/sync-frontend.sh`)." + ) + + # If the preferred UI port is taken, fall back to a free one. + probe_host = "127.0.0.1" if ui_host in ("0.0.0.0", "::", "") else ui_host + actual_port = ui_server.find_free_port(ui_port, host=probe_host) + if actual_port != ui_port: + logger.warning(f"Port {ui_port} is busy; using {actual_port} instead.") + ui_port = actual_port + logger.info(f"UI port source: {ui_port_source} (preferred {preferred_ui_port}, using {ui_port})") + os.environ["WL_LAST_UI_PORT"] = str(ui_port) + + ui_server.serve_ui( + ui_host=ui_host, + ui_port=ui_port, + backend_host=backend_host, + backend_port=backend_port, + open_browser=open_browser, + certs_dir=certs_dir, + grpc_auth_token=grpc_auth_token, + block=True, + ) + + +def _add_ui_server_flags(p: argparse.ArgumentParser) -> None: + """Attach flags for the native UI server.""" + p.add_argument('--port', type=int, default=None, + help='UI HTTP port (default: config ui_port, else $WL_LAST_UI_PORT, else 50051)') + p.add_argument('--config', default=None, + help='Experiment config file to read ui_port from (yaml/yml)') + p.add_argument('--host', default=None, + help='UI bind host (default: 0.0.0.0)') + p.add_argument('--backend-host', dest='backend_host', default=None, + help='Backend gRPC host to proxy to (default: localhost)') + p.add_argument('--backend-port', dest='backend_port', type=int, default=None, + help='Backend gRPC port to proxy to (default: 50051)') + p.add_argument('--no-browser', dest='no_browser', action='store_true', + help='Do not open the web browser automatically') + p.add_argument('--certs', action='store_true', + help='Serve HTTPS + mTLS to the backend using TLS certs from ' + '$WEIGHTSLAB_CERTS_DIR (default: unsecured HTTP). ' + 'Run `weightslab se` first to generate them.') + + +def _add_example_kind_flags(p: argparse.ArgumentParser) -> None: + """Attach the mutually-exclusive example-kind flags (default: classification).""" + group = p.add_mutually_exclusive_group() + group.add_argument("--cls", action="store_const", dest="example_kind", const="cls", + help="Run the classification example (default)") + group.add_argument("--seg", action="store_const", dest="example_kind", const="seg", + help="Run the segmentation example") + group.add_argument("--det", action="store_const", dest="example_kind", const="det", + help="Run the detection example") + group.add_argument("--clus", action="store_const", dest="example_kind", const="clus", + help="Run the clustering example") + group.add_argument("--gen", action="store_const", dest="example_kind", const="gen", + help="Run the generation example") + group.add_argument("--3d_det", action="store_const", dest="example_kind", const="3d_det", + help="Run the 3D LiDAR point-cloud detection example") + group.add_argument("--2d_det", action="store_const", dest="example_kind", const="2d_det", + help="Run the 2D LiDAR point-cloud detection example") + p.set_defaults(example_kind=_DEFAULT_EXAMPLE) + + +def _build_parser() -> argparse.ArgumentParser: + """Build the top-level argument parser (banner + detailed command reference). + + The CLI is intentionally minimal — exactly these commands: + weightslab --help | -h | help + weightslab se [--force-certs] + weightslab start [--port PORT] [--config FILE] [--backend-port PORT] [--certs] + weightslab start example [--cls|--seg|--det|--clus|--gen|--3d_det|--2d_det] + weightslab cli [--port PORT] [--host HOST] + weightslab tunnel ENDPOINT + """ + parser = argparse.ArgumentParser( + prog="weightslab", + description=_DESCRIPTION, + epilog=_EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command", metavar="{se,start,cli,tunnel,help}") + + # weightslab se [--force-certs] [certs_dir] + se_parser = sub.add_parser("se", help="Set up the secure environment (TLS certs + gRPC auth token)") + se_parser.add_argument('--force-certs', action='store_true', help='Regenerate certificates even if they already exist') + se_parser.add_argument('certs_dir', nargs='?', default=None, + help='Custom directory for certs/token (default: $WEIGHTSLAB_CERTS_DIR or ~/.weightslab-certs)') + + # weightslab cli [--port N] [--host H] + cli_parser = sub.add_parser( + "cli", help="Open an interactive terminal connected to the running experiment") + cli_parser.add_argument('--port', type=int, default=None, + help='CLI server port (default: auto-discover the running experiment)') + cli_parser.add_argument('--host', default=None, + help='CLI server host (default: localhost)') + + # weightslab tunnel ENDPOINT [--listen-port N] [--listen-host H] [--remote-port N] + tunnel_parser = sub.add_parser( + "tunnel", + help="Forward a remote gRPC backend (e.g. a Colab run via `ngrok tcp 50051`) " + "to a local port so `weightslab start` can proxy to it") + tunnel_parser.add_argument( + 'endpoint', nargs='?', default=None, + help="Remote backend endpoint host:port (e.g. bore.pub:12345). " + "A tcp:// prefix is accepted. Default: $WEIGHTSLAB_TUNNEL_ENDPOINT.") + tunnel_parser.add_argument( + '--listen-port', '-p', type=int, default=DEFAULT_LISTEN_PORT, + help=f"Local port to expose (default: {DEFAULT_LISTEN_PORT} — the port `weightslab start` proxies to)") + tunnel_parser.add_argument( + '--listen-host', default=None, + help="Local interface to bind (default: 127.0.0.1 on Windows/macOS, 0.0.0.0 on Linux)") + tunnel_parser.add_argument( + '--remote-port', type=int, default=None, + help="Remote port, if not included in ENDPOINT") + + # weightslab start [--port N ...] -> native UI server + # weightslab start example [--cls|--seg|--clus|--gen] -> bundled example + start_parser = sub.add_parser( + "start", + help="Start the Weights Studio UI natively (no Docker); " + "or `start example` to run a bundled example") + # Flags on the bare `start` command launch the native UI server. + _add_ui_server_flags(start_parser) + start_sub = start_parser.add_subparsers(dest="start_target") + example_parser = start_sub.add_parser( + "example", help="Start a bundled PyTorch example (default: classification)") + _add_example_kind_flags(example_parser) + + # Tolerate the swapped order: `weightslab example start [flags]` (and bare + # `weightslab example`) behave exactly like `weightslab start example`. Hidden + # from --help on purpose (argparse.SUPPRESS) — a forgiving fallback. + example_alias = sub.add_parser("example", help=argparse.SUPPRESS) + example_alias_sub = example_alias.add_subparsers(dest="example_action") + example_alias_start = example_alias_sub.add_parser( + "start", help="Start a bundled PyTorch example (default: classification)") + _add_example_kind_flags(example_alias_start) + + sub.add_parser("help", help="Show this help message") + + return parser + + +def main(): + parser = _build_parser() + args = parser.parse_args() + + if args.command == "help" or args.command is None: + parser.print_help() + elif args.command == "cli": + cli_connect(args) + elif args.command == "tunnel": + from weightslab.tunnel import tunnel_connect + tunnel_connect(args) + elif args.command == "se": + ui_secure_environment(args) + elif args.command == "start": + if getattr(args, "start_target", None) == "example": + example_start(args) + else: + # Bare `weightslab start` -> launch the native UI. + ui_start_native(args) + elif args.command == "example": + # Alias for `start example` — tolerate the swapped subcommand order. + example_start(args) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/weightslab/components/checkpoint_manager.py b/weightslab/components/checkpoint_manager.py index 0166a4f9..528d6e9d 100644 --- a/weightslab/components/checkpoint_manager.py +++ b/weightslab/components/checkpoint_manager.py @@ -113,6 +113,11 @@ def __init__(self, root_log_dir: str = 'root_experiment', load_model: bool = Tru self.data_checkpoint_dir.mkdir(exist_ok=True) self.loggers_dir.mkdir(exist_ok=True) + # Persist the signal-history logger to an on-disk DuckDB file under + # loggers/. Handles the ordering where the logger was created before + # this checkpoint manager (the reverse is handled in LoggerQueue.__init__). + self._bind_logger_to_disk() + # Manifest file for tracking hash chronology self.manifest_file = self.checkpoints_dir / "manifest.yaml" @@ -194,6 +199,31 @@ def _get_data_state_snapshot(self, dfm): except Exception: return None + def get_logger_db_path(self) -> Path: + """Path of the on-disk DuckDB file backing the signal-history logger.""" + return self.loggers_dir / "loggers.duckdb" + + def _bind_logger_to_disk(self) -> None: + """Point the registered logger at an on-disk DuckDB file, if possible. + + Safe no-op when no logger is registered yet or it predates on-disk + persistence support.""" + try: + lg = ledgers.get_logger() + if lg is not None and hasattr(lg, "set_db_path"): + lg.set_db_path(str(self.get_logger_db_path())) + except Exception as e: + logger.warning(f"Could not bind logger to on-disk DuckDB: {e}") + + def flush_logger_to_disk(self) -> None: + """Force the logger to checkpoint its DuckDB history to disk (if any).""" + try: + lg = ledgers.get_logger() + if lg is not None and hasattr(lg, "flush_to_disk"): + lg.flush_to_disk() + except Exception as e: + logger.warning(f"Could not flush logger to disk: {e}") + def _get_logger_snapshot_dir(self) -> Path: return self.loggers_dir @@ -871,9 +901,9 @@ def _save_changes( except Exception as e: logger.warning(f"Failed to save weights with component changes: {e}") - # Always save logger snapshot alongside other components (same hash) - if dump_model_architecture or dump_model_state or dump_optimizer_state or dump_config_state: - self.save_logger_snapshot() + # Durably checkpoint the on-disk signal-history DuckDB alongside the + # rest of the state (cheap CHECKPOINT; replaces the old JSON snapshot). + self.flush_logger_to_disk() def save_model_checkpoint( self, @@ -977,12 +1007,6 @@ def save_model_checkpoint( # If model architecture doesn't exist in this hash directory, save a reference to where it is if self.config.get('checkpoint_manager', {}).get('dump_model_architecture', False): self._save_architecture_reference_if_needed() # TODO (GP): Disable for now because it adds complexity for big models, and we want to ensure architecture is always saved with weights for simplicity - - # Persist logger queues alongside weight checkpoints - try: - self.save_logger_snapshot() - except Exception as e: - logger.debug(f"Could not save logger snapshot with checkpoint: {e}") return checkpoint_file except Exception as e: logger.error(f"Failed to save model checkpoint: {e}") @@ -1214,99 +1238,6 @@ def save_data_snapshot(self, force_new_state: bool = False) -> Optional[Path]: logger.error(f"Failed to save data snapshot: {e}") return None - def save_logger_snapshot(self, exp_hash: Optional[str] = None) -> Optional[Path]: - """Persist logger queues for the given experiment hash. - - Uses the same hash as model/hp/data; does not affect hashing. - """ - exp = exp_hash or self.current_exp_hash - if exp is None: - return None - - try: - logger_names = ledgers.list_loggers() - if not logger_names: - return None - - snapshot = {"exp_hash": exp, "timestamp": datetime.now().isoformat(), "loggers": {}} - for lname in logger_names: - lg = ledgers.get_logger(lname) - if lg == None: - continue - - # Get snapshot data from logger, using dedicated method if available for better encapsulation (e.g. for LoggerQueue) - payload = lg.save_snapshot() - - # Get final snapshot for this logger - snapshot["loggers"][lname] = payload - - if not snapshot["loggers"]: - return None - - snapshot_dir = self._get_logger_snapshot_dir() - snapshot_dir.mkdir(parents=True, exist_ok=True) - - # Merge existing payload to avoid dropping loggers not currently registered - try: - existing = self._load_logger_snapshot_payload() - existing_loggers = existing.get("loggers", {}) if isinstance(existing, dict) else {} - if existing_loggers: - recursive_update(existing_loggers, snapshot.get("loggers", {})) - snapshot["loggers"] = existing_loggers - except Exception as e: - logger.warning(f"Failed to merge existing logger snapshot: {e}") - - records: List[bytes] = [] - for lname, payload in snapshot["loggers"].items(): - line = json.dumps({"logger_name": lname, "payload": payload}, default=str) + "\n" - records.append(line.encode("utf-8")) - - chunks: List[bytes] = [] - current_chunk = bytearray() - for record in records: - if current_chunk and (len(current_chunk) + len(record) > self.LOGGER_SNAPSHOT_MAX_FILE_SIZE_BYTES): - chunks.append(bytes(current_chunk)) - current_chunk = bytearray() - current_chunk.extend(record) - if current_chunk: - chunks.append(bytes(current_chunk)) - - old_chunks = self._list_logger_snapshot_chunks() - for old_chunk in old_chunks: - try: - old_chunk.unlink() - except Exception: - pass - - compressor = zstd.ZstdCompressor(level=3) - chunk_names: List[str] = [] - for idx, raw_chunk in enumerate(chunks, start=1): - chunk_path = self._get_logger_snapshot_chunk_path(idx) - tmp_chunk_path = chunk_path.with_name(chunk_path.name + ".tmp") - with open(tmp_chunk_path, "wb") as f: - f.write(compressor.compress(raw_chunk)) - os.replace(tmp_chunk_path, chunk_path) - chunk_names.append(chunk_path.name) - - manifest = { - "exp_hash": exp, - "timestamp": datetime.now().isoformat(), - "format": "ndjson+zstd", - "max_file_size_bytes": self.LOGGER_SNAPSHOT_MAX_FILE_SIZE_BYTES, - "chunks": chunk_names, - } - manifest_path = self._get_logger_snapshot_manifest_path() - tmp_manifest_path = manifest_path.with_name(manifest_path.name + ".tmp") - with open(tmp_manifest_path, "w", encoding="utf-8") as f: - json.dump(manifest, f, indent=2) - os.replace(tmp_manifest_path, manifest_path) - - logger.info(f"Saved logger snapshot: {manifest_path} ({len(chunk_names)} chunks)") - return manifest_path - except Exception as e: - logger.warning(f"Failed to save logger snapshot: {e}") - return None - def save_pending_changes(self, force: bool = False) -> bool: """Dump any pending changes to disk. @@ -1473,6 +1404,36 @@ def _load_manager_state(self): except Exception as e: logger.warning(f"Failed to load manager state: {e}") + def _resolve_device(self, model: Optional[th.nn.Module] = None) -> th.device: + """Resolve the torch device to deserialize a checkpoint onto. + + Priority: the device of an already-loaded model, then the device + configured in the ledger's hyperparameters, then CPU. This avoids + ``torch.load`` defaulting to the checkpoint's original (possibly + CUDA) device on a machine where that device isn't available. + """ + if model is None: + try: + model = ledgers.get_model() + if model is not None and hasattr(model, 'get') and callable(model.get): + model = model.get() + except Exception: + model = None + if model is not None and hasattr(model, 'device'): + return model.device + + try: + configured_device = (ledgers.get_hyperparams() or {}).get('device') + except Exception: + configured_device = None + if configured_device: + try: + return th.device(configured_device) + except Exception: + logger.warning(f"Could not parse configured device {configured_device!r}; defaulting to CPU") + + return th.device('cpu') + def load_latest_checkpoint( self, model: Optional[th.nn.Module] = None, @@ -1516,7 +1477,8 @@ def load_latest_checkpoint( logger.info(f"Loading checkpoint: {latest_checkpoint.name}") try: - checkpoint = th.load(latest_checkpoint, weights_only=False) + device = self._resolve_device(model) + checkpoint = th.load(latest_checkpoint, weights_only=False, map_location=device) # Load model state if model is None: @@ -1685,7 +1647,7 @@ def load_checkpoint(self, if checkpoint_files: latest_checkpoint = checkpoint_files[-1] try: - checkpoint = th.load(latest_checkpoint, weights_only=False) + checkpoint = th.load(latest_checkpoint, weights_only=False, map_location=self._resolve_device()) rng_state = checkpoint.get('rng_state') if rng_state: result['rng_state'] = rng_state @@ -1725,7 +1687,10 @@ def load_checkpoint(self, if checkpoint_file_to_load: try: - result['weights'] = th.load(checkpoint_file_to_load, weights_only=False) + result['weights'] = th.load( + checkpoint_file_to_load, weights_only=False, + map_location=self._resolve_device(result.get('model')), + ) result['loaded_components'].add('weights') step = result['weights'].get('step', 0) diff --git a/weightslab/components/global_monitoring.py b/weightslab/components/global_monitoring.py index 4dcfa184..a9d9d124 100644 --- a/weightslab/components/global_monitoring.py +++ b/weightslab/components/global_monitoring.py @@ -1,12 +1,13 @@ import os -from typing import Any, Optional -from enum import Enum import contextvars - -from threading import Event import threading import time import logging +import traceback + +from typing import Any, Optional +from enum import Enum +from threading import Event from weightslab.backend.ledgers import get_hyperparams, set_hyperparam, resolve_hp_name, get_checkpoint_manager, get_model from weightslab.components.tracking import TrackingMode @@ -276,7 +277,7 @@ def __enter__(self, f: bool = False): self.model.set_tracking_mode(TrackingMode.EVAL) self.model.eval() - def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any, f: bool = False) -> bool: + def __exit__(self, exc_type: Any, exc_value: Any, traceback_: Any, f: bool = False) -> bool: """ Executed upon exiting the 'with' block (after user code runs). Reverts the model state. @@ -296,11 +297,11 @@ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any, f: bool = Fals if exc_type is RuntimeError: logger.debug(f"Suppressing exception: {exc_value} in GuardContext.__exit__:") - traceback.print_exc() if os.getenv("WL_DEBUG", "0") == "1" else None - self.architecture_guard.__exit__(exc_type, exc_value, traceback) + traceback.print_exc() if os.getenv("WEIGHTSLAB_LOG_LEVEL", "0") == "1" else None + self.architecture_guard.__exit__(exc_type, exc_value, traceback_) return True # suppress the exception - self.architecture_guard.__exit__(exc_type, exc_value, traceback) + self.architecture_guard.__exit__(exc_type, exc_value, traceback_) return False diff --git a/weightslab/data/data_samples_with_ops.py b/weightslab/data/data_samples_with_ops.py index 8236fe5f..d5ef1f5b 100644 --- a/weightslab/data/data_samples_with_ops.py +++ b/weightslab/data/data_samples_with_ops.py @@ -128,7 +128,7 @@ def __init__( wrapped_dataset: Dataset, root_log_dir: Optional[str] = None, is_training: bool = True, - compute_hash: bool = True, + compute_hash: bool = False, use_tags: bool = False, tags_mapping: Optional[Dict[str, int]] = None, stats_store: Optional[H5DataFrameStore] = None, diff --git a/weightslab/data/data_utils.py b/weightslab/data/data_utils.py index db8e7f2d..3f5aac51 100644 --- a/weightslab/data/data_utils.py +++ b/weightslab/data/data_utils.py @@ -1,3 +1,4 @@ +import io import re import numpy as np import torch as th @@ -618,6 +619,20 @@ def load_raw_image_array(dataset, index, rank: int = 0) -> tuple: if hasattr(wrapped, '__getitem__'): np_img, is_volumetric, original_shape = _get_image_array_and_metadata(wrapped, index, rank=rank) + # Tabular samples (1-D feature vectors) have no spatial dims and cannot be + # PIL-encoded as an image. Render a small heatmap for display continuity; + # the caller transmits the actual values via build_tabular_raw_data_stat. + if getattr(np_img, "ndim", None) == 1: + from weightslab.trainer.services.data_image_utils import render_tabular_heatmap + thumb_bytes, _shp = render_tabular_heatmap(np_img) + thumb_pil = None + if thumb_bytes: + try: + thumb_pil = Image.open(io.BytesIO(thumb_bytes)).convert("RGB") + except Exception: + thumb_pil = None + return np_img, False, original_shape, thumb_pil + # Point-cloud samples (LiDAR etc.) cannot be PIL-encoded; render a # server-side 2D thumbnail (BEV, range image, or custom projection). # Guarded by both the task type and a shape heuristic so regular image diff --git a/weightslab/data/dataframe_manager.py b/weightslab/data/dataframe_manager.py index 011c190f..e398e6fb 100644 --- a/weightslab/data/dataframe_manager.py +++ b/weightslab/data/dataframe_manager.py @@ -685,29 +685,39 @@ def upsert_df(self, df_local: List | pd.DataFrame, origin: str = None, force_flu if self._df.empty: self._df = df_norm.copy() else: - # Right-preferred upsert with multi-index support - existing_idx = df_norm.index.intersection(self._df.index) - all_cols = df_norm.columns - if len(existing_idx) > 0: - # Widen any categorical target columns to object before assigning: - # writing a value outside a Categorical's category list raises - # "Cannot setitem on a Categorical with a new category". This - # must include `origin` — re-registering rows whose (sample_id, - # annotation_id) already exist under a NEW origin hits exactly - # that error otherwise. The _optimize_dataframe_memory pass below - # re-applies categorical dtypes (origin included), so widening - # here is only transient and costs no memory afterward. - for col in all_cols: - if col in self._df.columns and isinstance(self._df[col].dtype, pd.CategoricalDtype): - self._df[col] = self._df[col].astype(object) - self._df.loc[existing_idx, all_cols] = df_norm.loc[existing_idx, all_cols] - - # Append rows that do not exist yet. Use a boolean mask (not - # .loc[difference]) so a duplicate key in df_norm can't be - # re-expanded into multiple rows by the label lookup. - new_rows = df_norm[~df_norm.index.isin(self._df.index)] - if not new_rows.empty: - self._df = pd.concat([self._df, new_rows]) + # Right-preferred upsert with multi-index support. + # If either side has duplicate index labels, label-based setitem + # can treat the replacement as a vector assignment into a scalar + # cell. Rebuild the overlap with concat instead of trying to write + # through .loc in that case. + if not self._df.index.is_unique or not df_norm.index.is_unique: + self._df = pd.concat([ + self._df.drop(index=df_norm.index, errors='ignore'), + df_norm[~df_norm.index.duplicated(keep='last')], + ]) + else: + existing_idx = df_norm.index.intersection(self._df.index) + all_cols = df_norm.columns + if len(existing_idx) > 0: + # Widen any categorical target columns to object before assigning: + # writing a value outside a Categorical's category list raises + # "Cannot setitem on a Categorical with a new category". This + # must include `origin` — re-registering rows whose (sample_id, + # annotation_id) already exist under a NEW origin hits exactly + # that error otherwise. The _optimize_dataframe_memory pass below + # re-applies categorical dtypes (origin included), so widening + # here is only transient and costs no memory afterward. + for col in all_cols: + if col in self._df.columns and isinstance(self._df[col].dtype, pd.CategoricalDtype): + self._df[col] = self._df[col].astype(object) + self._df.loc[existing_idx, all_cols] = df_norm.loc[existing_idx, all_cols] + + # Append rows that do not exist yet. Use a boolean mask (not + # .loc[difference]) so a duplicate key in df_norm can't be + # re-expanded into multiple rows by the label lookup. + new_rows = df_norm[~df_norm.index.isin(self._df.index)] + if not new_rows.empty: + self._df = pd.concat([self._df, new_rows]) logger.debug(f"[LedgeredDataFrameManager] Global DataFrame updated: {len(self._df)} rows, {len(self._df.columns)} columns. Index: {self._df.index.names}") @@ -2151,7 +2161,7 @@ def get_combined_df( return df - def get_collapse_annotations_to_samples_df(self, iid: str = None) -> pd.DataFrame: + def get_collapse_annotations_to_samples_df(self, df: pd.DataFrame | None = None) -> pd.DataFrame: """Collapse a (sample_id, annotation_id) multi-index df to one row per sample. The shared dataframe manager now expands every sample into one row per @@ -2182,8 +2192,16 @@ def get_collapse_annotations_to_samples_df(self, iid: str = None) -> pd.DataFram Returns a single-level ``sample_id``-indexed dataframe (origin stays a column). Dataframes that are not annotation-expanded are returned as-is. + + Args: + df: An already-materialized combined frame (as returned by + :meth:`get_combined_df`) to collapse. When ``None`` (default) a + fresh combined frame is pulled here. Callers that just pulled the + combined frame should pass it in to avoid a second full copy + + buffer merge + proxy conversion over the whole dataset. """ - df = self.get_combined_df() + if df is None: + df = self.get_combined_df() SID = SampleStatsEx.SAMPLE_ID.value ANNOT = SampleStatsEx.INSTANCE_ID.value @@ -2423,14 +2441,14 @@ def flush_if_needed_nonblocking(self, force: bool = False): self._buffer = {} if buffered: - logger.info(f"Flushing {len(buffered)} buffered records to DataFrame (non-blocking).") + logger.debug(f"Flushing {len(buffered)} buffered records to DataFrame (non-blocking).") self._apply_buffer_records_nonblocking(buffered) - logger.info(f"Applied {len(buffered)} buffered records to DataFrame (non-blocking).") + logger.debug(f"Applied {len(buffered)} buffered records to DataFrame (non-blocking).") # Always check H5 flush even when buffer was empty (pending rows must drain too). - logger.info(f"Checking if flush to H5 is needed (non-blocking). Pending count: {len(self._pending)}.") + logger.debug(f"Checking if flush to H5 is needed (non-blocking). Pending count: {len(self._pending)}.") self._flush_to_h5_if_needed(force=force) - logger.info(f"Completed non-blocking flush check. Pending count after flush: {len(self._pending)}.") + logger.debug(f"Completed non-blocking flush check. Pending count after flush: {len(self._pending)}.") def flush(self): """Blocking flush: buffer → DF → H5. @@ -2446,14 +2464,14 @@ def flush(self): # Step 2: apply to DF (blocking _lock acquisition; outside buffer lock). if buffered: - logger.info(f"Flushing {len(buffered)} buffered records to DataFrame.") + logger.debug(f"Flushing {len(buffered)} buffered records to DataFrame.") self._apply_buffer_records(buffered) - logger.info(f"Applied {len(buffered)} buffered records to DataFrame.") + logger.debug(f"Applied {len(buffered)} buffered records to DataFrame.") # Step 3: flush DF → H5 (outside buffer lock). - logger.info(f"Checking if flush to H5 is needed. Pending count: {len(self._pending)}.") + logger.debug(f"Checking if flush to H5 is needed. Pending count: {len(self._pending)}.") self._flush_to_h5_if_needed(force=True, blocking=True) - logger.info(f"Completed flush. Pending count after flush: {len(self._pending)}.") + logger.debug(f"Completed flush. Pending count after flush: {len(self._pending)}.") # Create global instance with config-driven parameters def create_ledger_manager(): diff --git a/weightslab/data/h5_array_store.py b/weightslab/data/h5_array_store.py index 346e3b86..02c3e7e3 100644 --- a/weightslab/data/h5_array_store.py +++ b/weightslab/data/h5_array_store.py @@ -470,39 +470,49 @@ def save_array( self._ensure_parent() + # Acquire the exclusive write side of the read-write lock so that no + # reader (load_array / load_arrays_batch) can have the file open in + # 'r' mode while we open it in 'a' mode. HDF5 forbids opening the same + # file read-write while it is already open read-only in the same + # process ("file is already open for read-only"), which otherwise + # corrupts the store. with self._local_lock: - with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, poll_interval=self._poll_interval): - try: - with h5py.File(str(self._path), 'a') as f: - # Create group structure: /sample_id/key_name/ - sample_group_name = str(sample_id) - if sample_group_name not in f: - sample_group = f.create_group(sample_group_name) - else: - sample_group = f[sample_group_name] - - # Remove existing key if present - if key_name in sample_group: - del sample_group[key_name] - - # Create key group - key_group = sample_group.create_group(key_name) - - # Store array data (with optional compression) - if self._use_compression: - key_group.create_dataset('data', data=array, compression='gzip', compression_opts=4) - else: - key_group.create_dataset('data', data=array) - - # Store metadata - for k, v in metadata.items(): - key_group.attrs[k] = v - - return self._build_path_reference(sample_id, key_name) - - except Exception as exc: - logger.error(f"[H5ArrayStore] Failed to save array for sample_id={sample_id}, key={key_name}: {exc}") - return None + self._rw_lock.acquire_write() + try: + with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, poll_interval=self._poll_interval): + try: + with h5py.File(str(self._path), 'a') as f: + # Create group structure: /sample_id/key_name/ + sample_group_name = str(sample_id) + if sample_group_name not in f: + sample_group = f.create_group(sample_group_name) + else: + sample_group = f[sample_group_name] + + # Remove existing key if present + if key_name in sample_group: + del sample_group[key_name] + + # Create key group + key_group = sample_group.create_group(key_name) + + # Store array data (with optional compression) + if self._use_compression: + key_group.create_dataset('data', data=array, compression='gzip', compression_opts=4) + else: + key_group.create_dataset('data', data=array) + + # Store metadata + for k, v in metadata.items(): + key_group.attrs[k] = v + + return self._build_path_reference(sample_id, key_name) + + except Exception as exc: + logger.error(f"[H5ArrayStore] Failed to save array for sample_id={sample_id}, key={key_name}: {exc}") + return None + finally: + self._rw_lock.release_write() def save_arrays_batch( self, @@ -571,43 +581,74 @@ def save_arrays_batch( return {} # --- Phase 2: merge temp into main file under lock --- + # Hold the exclusive write side of the read-write lock so that no + # reader (load_array / load_arrays_batch) has the file open in 'r' + # mode while we open it in 'a' mode. HDF5 refuses to open a file + # read-write while it is already open read-only in the same process + # ("file is already open for read-only"); that failure previously + # triggered a backup-restore that overwrote the live file out from + # under open reader handles and corrupted the store. with self._local_lock: - with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, poll_interval=self._poll_interval): - backup_path = self._create_backup() - try: - with h5py.File(str(self._path), 'a') as f_main: - with h5py.File(str(tmp_path), 'r') as f_tmp: - for sample_group_name in f_tmp: - if sample_group_name in f_main: - del f_main[sample_group_name] - f_tmp.copy(sample_group_name, f_main) - - path_refs: Dict[int, Dict[str, str]] = {} - for sample_group_name, key_data in prepared.items(): - sample_id = sample_group_name - path_refs[sample_id] = { - key_name: self._build_path_reference(sample_id, key_name) - for key_name in key_data - } - - # Merge succeeded — remove backup so recover() doesn't restore it - if backup_path: - backup_path.unlink(missing_ok=True) - - logger.debug( - f"[H5ArrayStore] Successfully upserted " - f"{sum(len(v) for v in path_refs.values())} arrays for " - f"{len(path_refs)} samples" - ) - return path_refs - - except Exception as exc: - logger.error(f"[H5ArrayStore] Failed to merge temp batch file: {exc}") - if backup_path: - self._restore_backup(backup_path) - return {} - finally: - tmp_path.unlink(missing_ok=True) + self._rw_lock.acquire_write() + try: + with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, poll_interval=self._poll_interval): + backup_path = self._create_backup() + # Track whether we actually began mutating the main file. + # If we never opened it (e.g. it was busy), the file is + # intact and restoring from a copy would only risk harm. + main_opened = False + try: + with h5py.File(str(self._path), 'a') as f_main: + main_opened = True + with h5py.File(str(tmp_path), 'r') as f_tmp: + for sample_group_name in f_tmp: + if sample_group_name not in f_main: + f_main.create_group(sample_group_name) + dest_group = f_main[sample_group_name] + src_group = f_tmp[sample_group_name] + # Replace only the keys present in this batch + # (e.g. "prediction") so sibling keys written + # by an earlier flush (e.g. "target") survive. + for key_name in src_group: + if key_name in dest_group: + del dest_group[key_name] + f_tmp.copy(f"{sample_group_name}/{key_name}", dest_group) + + path_refs: Dict[int, Dict[str, str]] = {} + for sample_group_name, key_data in prepared.items(): + sample_id = sample_group_name + path_refs[sample_id] = { + key_name: self._build_path_reference(sample_id, key_name) + for key_name in key_data + } + + # Merge succeeded — remove backup so recover() doesn't restore it + if backup_path: + backup_path.unlink(missing_ok=True) + + logger.debug( + f"[H5ArrayStore] Successfully upserted " + f"{sum(len(v) for v in path_refs.values())} arrays for " + f"{len(path_refs)} samples" + ) + return path_refs + + except Exception as exc: + logger.error(f"[H5ArrayStore] Failed to merge temp batch file: {exc}") + if backup_path: + if main_opened: + # We started writing into the main file; it may + # be half-merged, so roll back to the backup. + self._restore_backup(backup_path) + else: + # Main file was never touched — drop the backup + # rather than overwrite a healthy file. + backup_path.unlink(missing_ok=True) + return {} + finally: + tmp_path.unlink(missing_ok=True) + finally: + self._rw_lock.release_write() def recover(self) -> None: """ @@ -661,7 +702,7 @@ def load_array(self, path_ref: str) -> Optional[np.ndarray]: return None # Use read lock for concurrent read access (multiple threads can load in parallel) - # self._rw_lock.acquire_read() + self._rw_lock.acquire_read() try: try: with h5py.File(str(self._path), 'r', locking=False) as f: diff --git a/weightslab/data/h5_dataframe_store.py b/weightslab/data/h5_dataframe_store.py index 9e2d05cf..476b5655 100644 --- a/weightslab/data/h5_dataframe_store.py +++ b/weightslab/data/h5_dataframe_store.py @@ -208,16 +208,26 @@ def save_tag_registry(self, registry: dict) -> None: return self._ensure_parent() payload = json.dumps(self._tag_registry) - try: - with self._local_lock: - with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, poll_interval=self._poll_interval): - with pd.HDFStore(str(self._path), mode="a") as store: - if self._TAG_REGISTRY_KEY in store: - store.remove(self._TAG_REGISTRY_KEY) - store.put(self._TAG_REGISTRY_KEY, pd.DataFrame({"registry": [payload]}), format="table") - store.flush() - except Exception as e: - logger.debug(f"[H5DataFrameStore] Failed to save tag registry: {e}") + # Windows can transiently deny reopening a file another handle (this + # process's own backup copy, antivirus scanning, etc.) just released + # a moment ago -- a couple of retries rides out that window without + # giving up on a save that would otherwise succeed a beat later. + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + try: + with self._local_lock: + with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, poll_interval=self._poll_interval): + with pd.HDFStore(str(self._path), mode="a") as store: + if self._TAG_REGISTRY_KEY in store: + store.remove(self._TAG_REGISTRY_KEY) + store.put(self._TAG_REGISTRY_KEY, pd.DataFrame({"registry": [payload]}), format="table") + store.flush() + return + except Exception as e: + if attempt == max_attempts: + logger.debug(f"[H5DataFrameStore] Failed to save tag registry after {max_attempts} attempts: {e}") + else: + time.sleep(self._poll_interval * attempt) def load_tag_registry(self) -> dict: """Load the categorical tag registry from H5 into memory and return it.""" diff --git a/weightslab/docker/Dockerfile b/weightslab/docker/Dockerfile deleted file mode 100644 index 4e61aecb..00000000 --- a/weightslab/docker/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Weights Studio Frontend - Production Ready -# Adapts to certificate availability: uses HTTPS if certs present, falls back to HTTP - -FROM nginx:alpine - -ARG VITE_WL_ENABLE_GRPC_AUTH_TOKEN=0 -ARG VITE_GRPC_AUTH_TOKEN= - -# Remove default config -RUN rm /etc/nginx/conf.d/default.conf - -# Create a placeholder index.html for now -# When frontend source is available, replace with built assets -RUN mkdir -p /usr/share/nginx/html && \ - echo '' > /usr/share/nginx/html/index.html && \ - echo 'Weights Studio' >> /usr/share/nginx/html/index.html && \ - echo '

Weights Studio - Frontend Loading

' >> /usr/share/nginx/html/index.html && \ - echo '

Backend connection ready. Frontend source will be integrated here.

' >> /usr/share/nginx/html/index.html && \ - echo '' >> /usr/share/nginx/html/index.html - -# Copy entrypoint script that dynamically configures nginx based on available certs -COPY docker/nginx-entrypoint.sh /docker-entrypoint.sh -RUN chmod +x /docker-entrypoint.sh - -EXPOSE 80 443 - -ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/weightslab/docker/docker/.dockerignore b/weightslab/docker/docker/.dockerignore deleted file mode 100644 index 6b553fb2..00000000 --- a/weightslab/docker/docker/.dockerignore +++ /dev/null @@ -1,20 +0,0 @@ -# Docker -node_modules/ -npm-debug.log* -.npm -.env -.env.local -docker-artifact/ - -# IDE -.vscode/ -.idea/ - -# Build outputs -dist/ -build/ -*.log - -# OS -.DS_Store -Thumbs.db diff --git a/weightslab/docker/docker/.env.example b/weightslab/docker/docker/.env.example deleted file mode 100644 index acabf5cc..00000000 --- a/weightslab/docker/docker/.env.example +++ /dev/null @@ -1,67 +0,0 @@ -# Weights Studio Docker - Environment Configuration -# Copy this to .env and adjust values as needed - -# Port Configuration (single port works with HTTP or HTTPS) -VITE_PORT=5173 - -# Backend Configuration -WS_SERVER_HOST=localhost -WS_SERVER_PORT=8080 -WS_SERVER_PROTOCOL=https - -# gRPC Backend Port (for Envoy) -GRPC_BACKEND_PORT=50051 -ENVOY_PORT=8080 - -# Node Environment -NODE_ENV=production - -# Frontend HTTPS Configuration -# Set to 1 if you've generated certs and want HTTPS in the browser console logs -# Set to 0 for HTTP-only development mode -VITE_DEV_SERVER_HTTPS=0 - -# gRPC Auth Token (optional) -# Set VITE_WL_ENABLE_GRPC_AUTH_TOKEN=1 to enable auth -# Set VITE_GRPC_AUTH_TOKEN to the token value -# If not set, auth is disabled -VITE_WL_ENABLE_GRPC_AUTH_TOKEN=0 -VITE_GRPC_AUTH_TOKEN= - -# Alternative: load from environment variable -# GRPC_AUTH_TOKEN=your-auth-token-here - -# Certificate Directory -# Set to your certs path (e.g., C:\Users\YourName\.weightslab-certs on Windows or ~/.weightslab-certs on Linux/Mac) -# Leave empty to use local .weightslab-certs directory (no certs, HTTP mode) -WEIGHTSLAB_CERTS_DIR=.weightslab-certs - -# DEPLOYMENT SCENARIOS: -# -# 1. HTTP ONLY (No Certs, No Auth) -# VITE_WL_ENABLE_GRPC_AUTH_TOKEN=0 -# Leave certs/ directory empty -# Access: http://localhost:5173 -# -# 2. HTTPS + Auth -# VITE_WL_ENABLE_GRPC_AUTH_TOKEN=1 -# VITE_GRPC_AUTH_TOKEN= -# Place certificates in ./envoy/certs/ -# Access: https://localhost:5173 -# -# 3. HTTPS Only (No Auth) -# VITE_WL_ENABLE_GRPC_AUTH_TOKEN=0 -# Place certificates in ./envoy/certs/ -# Access: https://localhost:5173 -# -# Certificate Files Required (if using HTTPS): -# Automatically loaded from: ~/.weightslab-certs/ -# - envoy-server.crt -# - envoy-server.key -# - ca.crt -# -# If certs don't exist in ~/.weightslab-certs/, the system runs in HTTP-only mode. -# -# Generate certificates using: -# bash weightslab/ui/docker/utils/generate-certs.sh # Generate certs for ~/.weightslab-certs/ -# FORCE_CREATE=1 bash weightslab/ui/docker/utils/generate-certs.sh # Regenerate certs diff --git a/weightslab/docker/docker/DEPLOYMENT.md b/weightslab/docker/docker/DEPLOYMENT.md deleted file mode 100644 index 3bfe619a..00000000 --- a/weightslab/docker/docker/DEPLOYMENT.md +++ /dev/null @@ -1,271 +0,0 @@ -# Weights Studio Docker Deployment Guide - -This Docker setup automatically adapts to the availability of TLS certificates and gRPC auth tokens. All scenarios use a **single port (5173)** that works with HTTP or HTTPS depending on certificate availability. - -## Requirements - -- **Docker Engine** with the daemon running. -- **Docker Compose v2** (the `docker compose` CLI plugin) — *recommended*, or the - legacy **v1** standalone binary (`docker-compose`, **≥ 1.27**). `weightslab ui launch` - auto-detects whichever is installed. - -> **Note:** The manual `docker compose ...` commands in this guide assume Compose v2. -> If you are on v1, substitute the hyphenated form, e.g. `docker-compose -f docker-compose.yml up -d`. - -## Quick Start - -### Scenario 1: HTTP Only (No Certs, No Auth) - Development/Testing - -Fastest way to get running without certificates: - -```bash -# Copy the default environment -cp .env.example .env - -# Ensure certificates directory is empty -rm -rf ../envoy/certs/* - -# Run with production compose -docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d - -# Access at: http://localhost:5173 -``` - -The container will: -- Listen on port 443 in HTTP mode -- Disable gRPC auth token validation -- Host port 5173 maps to container port 443 - -### Scenario 2: HTTPS + gRPC Auth - Secure Production - -```bash -# Generate certificates and auth token (one-time setup) -weightslab se - -# Copy environment with auth enabled -cp .env.example .env -cat >> .env << 'EOF' -VITE_WL_ENABLE_GRPC_AUTH_TOKEN=1 -EOF - -# Run with production compose -docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d - -# Access at: https://localhost:5173 -# Frontend will validate gRPC auth token on API calls -``` - -The container will: -- Detect certificates in `/etc/nginx/certs/` -- Enable HTTPS on port 443 -- Host port 5173 maps to container port 443 (HTTPS) -- Pass gRPC auth token to frontend - -### Scenario 3: HTTPS Only (No Auth) - -```bash -# Generate certificates without auth -weightslab se --no-auth - -# Copy environment with HTTPS, no auth -cp .env.example .env - -# Run with production compose -docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d - -# Access at: https://localhost:5173 -``` - -The container will: -- Detect certificates -- Enable HTTPS on port 443 -- Disable gRPC auth token validation -- Host port 5173 maps to container port 443 (HTTPS) - -## How It Works - -### Dynamic Configuration - -The Docker image includes an entrypoint script (`nginx-entrypoint.sh`) that: - -1. **Checks for certificates** at container startup: - - Looks for `/etc/nginx/certs/envoy-server.crt` and `.key` - -2. **Generates appropriate nginx server block**: - - Single server listening on port 443 - - If certs found → Enable `ssl` directive + certificate paths - - If certs missing → Plain HTTP on port 443 - -3. **Validates the config** and starts nginx - -### Volume Mounts - -The `docker-compose.yml` mounts: - -```yaml -volumes: - - ../envoy/certs:/etc/nginx/certs:ro # Optional - uses HTTP if empty or missing -``` - -The mount path is **optional**: -- If `../envoy/certs/` exists but is empty → HTTP on port 443 -- If certificates files exist → HTTPS on port 443 -- If directory doesn't exist → Docker creates it (empty) → HTTP on port 443 - -### Certificate Files - -When certificates are available, they must be at: - -``` -weightslab/ui/envoy/certs/ -├── envoy-server.crt -└── envoy-server.key -``` - -Generate with: - -```bash -weightslab se [--force-certs] [--no-auth] -``` - -## Environment Variables - -### Frontend Configuration - -| Variable | Default | Effect | -|----------|---------|--------| -| `VITE_WL_ENABLE_GRPC_AUTH_TOKEN` | `0` | Enable gRPC auth validation on frontend | -| `VITE_GRPC_AUTH_TOKEN` | empty | Auth token sent with gRPC calls | -| `WS_SERVER_HOST` | `localhost` | Backend server address | -| `WS_SERVER_PORT` | `8080` | Backend server port | -| `WS_SERVER_PROTOCOL` | `https` | Backend protocol (http/https) | - -### Docker Configuration - -| Variable | Default | Effect | -|----------|---------|--------| -| `VITE_PORT` | `5173` | Host port (container always uses 443) | -| `GRPC_BACKEND_PORT` | `50051` | Backend gRPC port for Envoy | -| `ENVOY_PORT` | `8080` | Envoy gRPC-Web proxy port | -| `NODE_ENV` | `production` | Node environment | - -## Health Checks - -The nginx config includes a health endpoint at `/health`: - -```bash -# HTTP mode (no certs) -curl http://localhost:5173/health -# Output: healthy - -# HTTPS mode (with certs) -curl https://localhost:5173/health -# Output: healthy (ignoring cert warnings in curl: curl -k) -``` - -## Troubleshooting - -### Container won't start - -1. Check logs: - ```bash - docker compose logs weights_studio - ``` - -2. Common issues: - - Certs directory doesn't exist → Create: `mkdir -p ../envoy/certs` - - Port conflict → Check `docker ps` or change `VITE_HTTP_PORT`/`VITE_HTTPS_PORT` - -### HTTPS not working - -1. Verify certificates exist: - ```bash - ls -la ../envoy/certs/ - ``` - -2. Check nginx accepted the config: - ```bash - docker exec weights_studio_frontend nginx -t - ``` - -### gRPC calls failing - -1. Check Envoy is running: - ```bash - docker ps | grep envoy - ``` - -2. Verify backend is reachable: - ```bash - docker exec weights_studio_frontend curl -v http://weights_studio_envoy:8080/ - ``` - -3. Validate auth token if enabled: - - Frontend sends `Authorization: Bearer ` header - - Backend must accept the same token - -## Production Deployment - -### Pre-deployment Checklist - -- [ ] Generate TLS certificates (recommended for production) -- [ ] Set gRPC auth token (if exposing externally) -- [ ] Configure `WS_SERVER_HOST` to point to actual backend -- [ ] Test health endpoints -- [ ] Verify backend connectivity -- [ ] Review nginx logs after startup - -### Recommended Configuration - -```bash -# Production with full security -weightslab se --force-certs # Generate fresh certs - -# Set environment -cat > .env << 'EOF' -VITE_WL_ENABLE_GRPC_AUTH_TOKEN=1 -WS_SERVER_HOST=backend.production.internal -WS_SERVER_PORT=8080 -WS_SERVER_PROTOCOL=https -NODE_ENV=production -EOF - -# Deploy -docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d -``` - -### Scaling - -To run multiple instances: - -1. Change container names and ports per instance: - ```bash - VITE_HTTP_PORT=5174 VITE_HTTPS_PORT=5175 docker compose ... up -d - ``` - -2. Use a load balancer in front -3. Share the same certificates and auth token across instances -4. Ensure all point to the same backend - -## Docker Image - -### Building - -```bash -# From weightslab/ui/ directory -docker build -t weights_studio_frontend:latest . -``` - -### Publishing - -```bash -docker tag weights_studio_frontend:latest graybx/weightslab:latest -docker push graybx/weightslab:latest -``` - -Then update `docker-compose.yml`: -```yaml -services: - weights_studio: - image: graybx/weightslab:latest -``` diff --git a/weightslab/docker/docker/docker-compose.yml b/weightslab/docker/docker/docker-compose.yml deleted file mode 100644 index e12f65e3..00000000 --- a/weightslab/docker/docker/docker-compose.yml +++ /dev/null @@ -1,127 +0,0 @@ -# Global Docker Compose configuration for Weights Studio -services: - # Envoy proxy for gRPC-Web - envoy: - image: envoyproxy/envoy:v1.28-latest - container_name: weights_studio_envoy - ports: - - "${ENVOY_PORT:-8080}:${ENVOY_PORT:-8080}" - - "9901:9901" - volumes: - - ./../envoy/envoy.yaml:/etc/envoy/envoy.yaml:ro - - ./../envoy/envoy.upstream_plaintext.yaml:/etc/envoy/envoy.upstream_plaintext.yaml:ro - - ./../envoy/envoy.downstream_plaintext.yaml:/etc/envoy/envoy.downstream_plaintext.yaml:ro - - ./../envoy/envoy.downstream_upstream_plaintext.yaml:/etc/envoy/envoy.downstream_upstream_plaintext.yaml:ro - - ${WEIGHTSLAB_CERTS_DIR:-${HOME}/.weightslab-certs}:/etc/envoy/certs:ro - extra_hosts: - - "host.docker.internal:host-gateway" - - "grpc-backend:host-gateway" - environment: - - GRPC_BACKEND_PORT=${GRPC_BACKEND_PORT:-50051} - - ENVOY_UPSTREAM_TLS=${ENVOY_UPSTREAM_TLS:-auto} - - ENVOY_DOWNSTREAM_TLS=${ENVOY_DOWNSTREAM_TLS:-auto} - command: - - /bin/sh - - -c - - | - GRPC_BACKEND_PORT="$${GRPC_BACKEND_PORT:-50051}"; - ENVOY_UPSTREAM_TLS="$${ENVOY_UPSTREAM_TLS:-auto}"; - ENVOY_DOWNSTREAM_TLS="$${ENVOY_DOWNSTREAM_TLS:-auto}"; - ENVOY_UPSTREAM_EFFECTIVE="$${ENVOY_UPSTREAM_TLS}"; - ENVOY_DOWNSTREAM_EFFECTIVE="$${ENVOY_DOWNSTREAM_TLS}"; - if [ "$${ENVOY_UPSTREAM_EFFECTIVE}" = "auto" ]; then - if [ -f /etc/envoy/certs/envoy-client.crt ] && [ -f /etc/envoy/certs/envoy-client.key ] && [ -f /etc/envoy/certs/ca.crt ]; then - ENVOY_UPSTREAM_EFFECTIVE="on"; - else - ENVOY_UPSTREAM_EFFECTIVE="off"; - fi; - fi; - if [ "$${ENVOY_DOWNSTREAM_EFFECTIVE}" = "auto" ]; then - if [ -f /etc/envoy/certs/envoy-server.crt ] && [ -f /etc/envoy/certs/envoy-server.key ]; then - ENVOY_DOWNSTREAM_EFFECTIVE="on"; - else - ENVOY_DOWNSTREAM_EFFECTIVE="off"; - fi; - fi; - if [ "$${ENVOY_DOWNSTREAM_EFFECTIVE}" = "on" ] && [ "$${ENVOY_UPSTREAM_EFFECTIVE}" = "on" ]; then - ENVOY_TEMPLATE="/etc/envoy/envoy.yaml"; - elif [ "$${ENVOY_DOWNSTREAM_EFFECTIVE}" = "on" ] && [ "$${ENVOY_UPSTREAM_EFFECTIVE}" = "off" ]; then - ENVOY_TEMPLATE="/etc/envoy/envoy.upstream_plaintext.yaml"; - elif [ "$${ENVOY_DOWNSTREAM_EFFECTIVE}" = "off" ] && [ "$${ENVOY_UPSTREAM_EFFECTIVE}" = "on" ]; then - ENVOY_TEMPLATE="/etc/envoy/envoy.downstream_plaintext.yaml"; - else - ENVOY_TEMPLATE="/etc/envoy/envoy.downstream_upstream_plaintext.yaml"; - fi; - echo "[envoy] downstream TLS=$${ENVOY_DOWNSTREAM_EFFECTIVE} upstream TLS=$${ENVOY_UPSTREAM_EFFECTIVE}"; - echo "[envoy] using template $${ENVOY_TEMPLATE}"; - sed "s/__GRPC_BACKEND_PORT__/$${GRPC_BACKEND_PORT}/g" "$${ENVOY_TEMPLATE}" > /tmp/envoy.generated.yaml; - exec /usr/local/bin/envoy -c /tmp/envoy.generated.yaml - healthcheck: - test: ["CMD", "sh", "-c", "kill -0 1"] - interval: 10s - timeout: 3s - retries: 3 - start_period: 10s - networks: - - weights_studio_network - restart: unless-stopped - - # Weights Studio frontend (production static server) - weights_studio: - image: ${WS_FRONTEND_IMAGE:-graybx/weightslab:latest} - container_name: weights_studio_frontend - build: - context: .. - dockerfile: Dockerfile - args: - VITE_IS_A_SANDBOX: ${IS_A_SANDBOX:-0} - VITE_HISTOGRAM_MAX_BINS: ${VITE_HISTOGRAM_MAX_BINS:-512} - # Token is injected at runtime, not at build time - VITE_WL_ENABLE_GRPC_AUTH_TOKEN: 0 - VITE_SERVER_PROTOCOL: ${VITE_SERVER_PROTOCOL:-https} - VITE_DEV_SERVER_HTTPS: ${VITE_DEV_SERVER_HTTPS:-0} - ports: - - "${VITE_HTTPS_PORT:-5173}:443" - environment: - - NODE_ENV=${NODE_ENV:-production} - - WS_SERVER_HOST=${WS_SERVER_HOST:-localhost} - - WS_SERVER_PORT=${WS_SERVER_PORT:-8080} - - # TLS settings for frontend to communicate with Envoy - - WS_SERVER_PROTOCOL=${WS_SERVER_PROTOCOL:-https} - - VITE_DEV_SERVER_HTTPS=${VITE_DEV_SERVER_HTTPS:-0} - - VITE_WL_ENABLE_GRPC_AUTH_TOKEN=${VITE_WL_ENABLE_GRPC_AUTH_TOKEN:-0} - - VITE_SERVER_PROTOCOL=${VITE_SERVER_PROTOCOL:-https} - - VITE_GRPC_AUTH_TOKEN=${VITE_GRPC_AUTH_TOKEN} - - WEIGHTSLAB_CERTS_DIR=${WEIGHTSLAB_CERTS_DIR:-${HOME}/.weightslab-certs} - - ENVOY_UPSTREAM_TLS=${ENVOY_UPSTREAM_TLS:-auto} - - ENVOY_DOWNSTREAM_TLS=${ENVOY_DOWNSTREAM_TLS:-auto} - - GRPC_BACKEND_PORT=${GRPC_BACKEND_PORT:-50051} - # Bounding-box render caps (runtime-overridable without a rebuild). - - BB_THUMB_RENDER=${BB_THUMB_RENDER:-10} - - BB_MODAL_RENDER=${BB_MODAL_RENDER:-100} - # Grid / cache tuning (runtime-overridable without a rebuild). - - GRID_WINDOW_SIZE=${GRID_WINDOW_SIZE:-6} - - GRID_CACHE_MAX_MB=${GRID_CACHE_MAX_MB:-128} - - MODAL_CACHE_MAX_MB=${MODAL_CACHE_MAX_MB:-64} - # Point-cloud rendering (PC_MAX_POINTS empty = 1 500 000 default). - - PC_MAX_POINTS=${PC_MAX_POINTS:-} - - DISABLE_GPU_RENDERING=${DISABLE_GPU_RENDERING:-0} - # Feature toggles (runtime-overridable; each defaults to 1 = enabled). - - ENABLE_PLOTS=${ENABLE_PLOTS:-1} - - ENABLE_DATA_EXPLORATION=${ENABLE_DATA_EXPLORATION:-1} - - ENABLE_HYPERPARAMETERS_OPTIMIZATION=${ENABLE_HYPERPARAMETERS_OPTIMIZATION:-1} - - ENABLE_AGENT=${ENABLE_AGENT:-1} - volumes: - # Mount certs directory if available - - ${WEIGHTSLAB_CERTS_DIR:-${HOME}/.weightslab-certs}:/etc/nginx/certs:ro - depends_on: - envoy: - condition: service_healthy - networks: - - weights_studio_network - restart: unless-stopped - -networks: - weights_studio_network: - driver: bridge diff --git a/weightslab/docker/docker/nginx-entrypoint.sh b/weightslab/docker/docker/nginx-entrypoint.sh deleted file mode 100644 index 259f89ef..00000000 --- a/weightslab/docker/docker/nginx-entrypoint.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/bin/sh -# Dynamic nginx configuration based on certificate availability -# Adapts to HTTPS if certs exist, falls back to HTTP if missing -# Injects gRPC auth token at runtime - -set -e - -WEIGHTSLAB_CERTS_DIR="/etc/nginx/certs" -CERT_FILE="$WEIGHTSLAB_CERTS_DIR/envoy-server.crt" -KEY_FILE="$WEIGHTSLAB_CERTS_DIR/envoy-server.key" -NGINX_CONF="/etc/nginx/conf.d/default.conf" - -# Check if certificates exist -HAS_CERTS=false -if [ -f "$CERT_FILE" ] && [ -f "$KEY_FILE" ]; then - HAS_CERTS=true - echo "✓ TLS certificates found - enabling HTTPS" -else - echo "✗ TLS certificates not found - using HTTP" -fi - -# Create config directory for token injection -mkdir -p /tmp/nginx_conf - -# Inject token into JavaScript config file that will be served -TOKEN_FILE="/tmp/nginx_conf/grpc-config.js" -cat > "$TOKEN_FILE" << TOKEN_END -// Runtime gRPC configuration (injected at container start) -window.__grpcConfig = { - authToken: '${GRPC_AUTH_TOKEN:-}', - authEnabled: ${WL_ENABLE_GRPC_AUTH_TOKEN:-0}, - protocol: '${WS_SERVER_PROTOCOL:-https}' -}; -TOKEN_END - -echo "✓ Runtime gRPC config injected" - -# Generate nginx configuration based on certificate availability -if [ "$HAS_CERTS" = true ]; then - # HTTPS configuration - cat > "$NGINX_CONF" << 'NGINX_HTTPS_END' -upstream envoy { - server envoy:8080; -} - -server { - listen 80; - listen 443 ssl; - server_name _; - - ssl_certificate /etc/nginx/certs/envoy-server.crt; - ssl_certificate_key /etc/nginx/certs/envoy-server.key; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers HIGH:!aNULL:!MD5; - ssl_prefer_server_ciphers on; - ssl_session_cache shared:SSL:10m; - ssl_session_timeout 10m; - - root /usr/share/nginx/html; - index index.html; - - # Cache static assets - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ { - expires 1y; - add_header Cache-Control "public, immutable"; - } - - # API proxy to backend through Envoy - location /api/ { - proxy_pass http://envoy/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # gRPC proxy - location ~ ^/grpc { - proxy_pass http://envoy; - proxy_http_version 1.1; - proxy_set_header Content-Type application/grpc; - proxy_set_header grpc-encoding gzip; - proxy_buffering off; - proxy_request_buffering off; - proxy_set_header Host $host; - } - - # SPA fallback - location / { - try_files $uri $uri/ /index.html; - } - - # Serve runtime gRPC config - location /grpc-config.js { - alias /tmp/nginx_conf/grpc-config.js; - add_header Content-Type application/javascript; - add_header Cache-Control "no-cache"; - } - - # Health check - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } -} -NGINX_HTTPS_END -else - # HTTP configuration - cat > "$NGINX_CONF" << 'NGINX_HTTP_END' -upstream envoy { - server envoy:8080; -} - -server { - listen 80; - listen 443; - server_name _; - - root /usr/share/nginx/html; - index index.html; - - # Cache static assets - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ { - expires 1y; - add_header Cache-Control "public, immutable"; - } - - # API proxy to backend through Envoy - location /api/ { - proxy_pass http://envoy/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # gRPC proxy - location ~ ^/grpc { - proxy_pass http://envoy; - proxy_http_version 1.1; - proxy_set_header Content-Type application/grpc; - proxy_set_header grpc-encoding gzip; - proxy_buffering off; - proxy_request_buffering off; - proxy_set_header Host $host; - } - - # SPA fallback - location / { - try_files $uri $uri/ /index.html; - } - - # Serve runtime gRPC config - location /grpc-config.js { - alias /tmp/nginx_conf/grpc-config.js; - add_header Content-Type application/javascript; - add_header Cache-Control "no-cache"; - } - - # Health check - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } -} -NGINX_HTTP_END -fi - -# Give envoy time to register in DNS and become ready -echo "Waiting for envoy service to be ready..." -sleep 2 -echo "✓ Proceeding with nginx configuration" - -# Validate nginx configuration -echo "Validating nginx configuration..." -nginx -t - -# Start nginx in foreground -echo "Starting nginx..." -exec nginx -g "daemon off;" diff --git a/weightslab/docker/docker/nginx.base-path.conf copy.template b/weightslab/docker/docker/nginx.base-path.conf copy.template deleted file mode 100644 index 07be634d..00000000 --- a/weightslab/docker/docker/nginx.base-path.conf copy.template +++ /dev/null @@ -1,40 +0,0 @@ -server { - listen 80; - server_name _; - - root /usr/share/nginx/html; - index index.html; - - # When accessed directly (e.g. port-forward to :5173), redirect to app root. - location = / { - return 301 ${NGINX_BASE_PATH}; - } - - # API proxy: forward gRPC-Web calls to Envoy. - # Strips the /{case}/api/ prefix before forwarding so Envoy receives the bare RPC path. - # Works for both direct port-forward access AND when behind Caddy. - location ^~ ${NGINX_BASE_PATH}api/ { - proxy_pass http://${NGINX_ENVOY_HOST}:${NGINX_ENVOY_PORT}/; - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_buffering off; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - } - - # Serve the SPA index at the exact base path. - location = ${NGINX_BASE_PATH} { - try_files /index.html =404; - } - - # Strip the base-path prefix and serve every other asset from the html root. - # Vite builds absolute asset URLs like /mnist/assets/app-xyz.js; - # this regex captures everything after the prefix and resolves it to - # /usr/share/nginx/html/, falling back to /index.html for SPA routes. - location ~* ^${NGINX_BASE_PATH}(.+)$ { - try_files /$1 /index.html; - add_header Cache-Control "public, max-age=604800, immutable" always; - } -} diff --git a/weightslab/docker/docker/nginx.base-path.conf.template b/weightslab/docker/docker/nginx.base-path.conf.template deleted file mode 100644 index 07be634d..00000000 --- a/weightslab/docker/docker/nginx.base-path.conf.template +++ /dev/null @@ -1,40 +0,0 @@ -server { - listen 80; - server_name _; - - root /usr/share/nginx/html; - index index.html; - - # When accessed directly (e.g. port-forward to :5173), redirect to app root. - location = / { - return 301 ${NGINX_BASE_PATH}; - } - - # API proxy: forward gRPC-Web calls to Envoy. - # Strips the /{case}/api/ prefix before forwarding so Envoy receives the bare RPC path. - # Works for both direct port-forward access AND when behind Caddy. - location ^~ ${NGINX_BASE_PATH}api/ { - proxy_pass http://${NGINX_ENVOY_HOST}:${NGINX_ENVOY_PORT}/; - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_buffering off; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - } - - # Serve the SPA index at the exact base path. - location = ${NGINX_BASE_PATH} { - try_files /index.html =404; - } - - # Strip the base-path prefix and serve every other asset from the html root. - # Vite builds absolute asset URLs like /mnist/assets/app-xyz.js; - # this regex captures everything after the prefix and resolves it to - # /usr/share/nginx/html/, falling back to /index.html for SPA routes. - location ~* ^${NGINX_BASE_PATH}(.+)$ { - try_files /$1 /index.html; - add_header Cache-Control "public, max-age=604800, immutable" always; - } -} diff --git a/weightslab/docker/docker/nginx.conf b/weightslab/docker/docker/nginx.conf deleted file mode 100644 index 82b67ea6..00000000 --- a/weightslab/docker/docker/nginx.conf +++ /dev/null @@ -1,57 +0,0 @@ -upstream backend { - server host.docker.internal:50051; -} - -upstream envoy { - server weightslab_envoy:8080; -} - -# HTTP server on port 443 (no certs available) -server { - listen 443; - server_name _; - - root /usr/share/nginx/html; - index index.html; - - # Cache static assets - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ { - expires 1y; - add_header Cache-Control "public, immutable"; - } - - # API proxy to backend through Envoy - location /api/ { - proxy_pass http://envoy/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # gRPC proxy - location ~ ^/grpc { - proxy_pass http://envoy; - proxy_http_version 1.1; - proxy_set_header Content-Type application/grpc; - proxy_set_header grpc-encoding gzip; - proxy_buffering off; - proxy_request_buffering off; - proxy_set_header Host $host; - } - - # SPA fallback - location / { - try_files $uri $uri/ /index.html; - } - - # Health check - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } -} diff --git a/weightslab/docker/docker/test-deployment.sh b/weightslab/docker/docker/test-deployment.sh deleted file mode 100644 index c8b71b37..00000000 --- a/weightslab/docker/docker/test-deployment.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# Test script to verify Weights Studio deployment in different modes - -set -e - -DOCKER_COMPOSE_CMD="docker compose -f docker-compose.yml -f docker-compose.prod.yml" -CONTAINER_NAME="weights_studio_frontend" - -echo "======================================" -echo "Weights Studio Deployment Test" -echo "======================================" - -# Function to wait for container -wait_for_container() { - local container=$1 - local max_attempts=30 - local attempt=0 - - echo "Waiting for $container to start..." - while [ $attempt -lt $max_attempts ]; do - if docker exec "$container" nginx -t 2>/dev/null; then - echo "✓ $container is running" - return 0 - fi - attempt=$((attempt + 1)) - sleep 1 - done - - echo "✗ $container failed to start" - return 1 -} - -# Function to test health endpoint -test_health() { - local port=$1 - local protocol=${2:-http} - - echo "Testing health endpoint on $protocol://localhost:$port..." - - if [ "$protocol" = "https" ]; then - if curl -k --silent "https://localhost:$port/health" 2>/dev/null | grep -q "healthy"; then - echo "✓ HTTPS health check passed" - return 0 - fi - else - if curl --silent "http://localhost:$port/health" 2>/dev/null | grep -q "healthy"; then - echo "✓ HTTP health check passed" - return 0 - fi - fi - - echo "✗ Health check failed" - return 1 -} - -# Function to check certificate status -check_certs() { - if [ -f "../envoy/certs/envoy-server.crt" ] && [ -f "../envoy/certs/envoy-server.key" ]; then - echo "✓ Certificates found" - return 0 - else - echo "✗ Certificates not found (HTTP-only mode)" - return 1 - fi -} - -# Main test flow -echo "" -echo "1. Checking certificate status..." -check_certs -HAVE_CERTS=$? - -echo "" -echo "2. Starting Docker stack..." -$DOCKER_COMPOSE_CMD up -d - -echo "" -echo "3. Waiting for services to start..." -wait_for_container "$CONTAINER_NAME" - -echo "" -echo "4. Checking container configuration..." -docker logs "$CONTAINER_NAME" | grep -E "✓|✗" | head -5 - -echo "" -echo "5. Testing HTTP(s) endpoint..." -test_health "5173" "http" - -echo "" -echo "7. Checking Envoy status..." -if docker exec weights_studio_envoy ps aux | grep -q "envoy"; then - echo "✓ Envoy proxy is running" -else - echo "✗ Envoy proxy failed to start" -fi - -echo "" -echo "======================================" -echo "Deployment test completed!" -echo "======================================" -echo "" -echo "Access points:" -echo " HTTP(s): http://localhost:5173" -echo "" -echo "To stop:" -echo " $DOCKER_COMPOSE_CMD down" -echo "" -echo "To view logs:" -echo " docker logs weights_studio_frontend" -echo " docker logs weights_studio_envoy" diff --git a/weightslab/docker/docker/utils/build-and-deploy.sh b/weightslab/docker/docker/utils/build-and-deploy.sh deleted file mode 100644 index b00371d1..00000000 --- a/weightslab/docker/docker/utils/build-and-deploy.sh +++ /dev/null @@ -1,348 +0,0 @@ -#!/bin/bash -# Auto-detect TLS and tokens, then build production image - -# Print all environment variables to diagnose issues -echo "===== Environment Variables Received =====" -env | grep -E 'WEIGHTSLAB|VITE|ENVOY|HOME' || true -echo "==========================================" - -# Ensure environment variables are available (important when called from Python subprocess) -echo "Init: WEIGHTSLAB_CERTS_DIR='$WEIGHTSLAB_CERTS_DIR' (received from env)" -WEIGHTSLAB_CERTS_DIR="${WEIGHTSLAB_CERTS_DIR:-$HOME/.weightslab-certs}" -export WEIGHTSLAB_CERTS_DIR -echo "After default: WEIGHTSLAB_CERTS_DIR='$WEIGHTSLAB_CERTS_DIR'" - -# Parse command line arguments -DEV=false -FORCE_UNSECURE=0 - -for arg in "$@"; do - case "$arg" in - --dev) - DEV=true - ;; - --unsecure|--unsecured) - echo "Forcing UNSECURE mode (HTTP, no auth)" - FORCE_UNSECURE=1 - ;; - ENABLE_PLOTS=*) ENABLE_PLOTS="${arg#*=}" ;; - ENABLE_DATA_EXPLORATION=*) ENABLE_DATA_EXPLORATION="${arg#*=}" ;; - ENABLE_HYPERPARAMETERS_OPTIMIZATION=*) ENABLE_HYPERPARAMETERS_OPTIMIZATION="${arg#*=}" ;; - ENABLE_AGENT=*) ENABLE_AGENT="${arg#*=}" ;; - BB_THUMB_RENDER=*) BB_THUMB_RENDER="${arg#*=}" ;; - BB_MODAL_RENDER=*) BB_MODAL_RENDER="${arg#*=}" ;; - WS_HISTOGRAM_MAX_BINS=*) WS_HISTOGRAM_MAX_BINS="${arg#*=}" ;; - GRID_WINDOW_SIZE=*) GRID_WINDOW_SIZE="${arg#*=}" ;; - GRID_CACHE_MAX_MB=*) GRID_CACHE_MAX_MB="${arg#*=}" ;; - MODAL_CACHE_MAX_MB=*) MODAL_CACHE_MAX_MB="${arg#*=}" ;; - PC_MAX_POINTS=*) PC_MAX_POINTS="${arg#*=}" ;; - DISABLE_GPU_RENDERING=*) DISABLE_GPU_RENDERING="${arg#*=}" ;; - esac -done - -# Check if WEIGHTSLAB_CERTS_DIR was explicitly set to empty string (before applying defaults) -# This is how the E2E tests disable certs: WEIGHTSLAB_CERTS_DIR="" -if [ "$FORCE_UNSECURE" = "1" ]; then - # Force HTTP / no auth, but KEEP WEIGHTSLAB_CERTS_DIR: the compose bind-mount - # needs a real (possibly empty) source directory. TLS/auth are forced off in - # the derivation block below regardless of any files in it. - echo "--unsecured flag provided: forcing HTTP / no auth (certs dir kept for bind-mount only)" -elif [ "${WEIGHTSLAB_CERTS_DIR+x}" ] && [ -z "$WEIGHTSLAB_CERTS_DIR" ]; then - echo "WEIGHTSLAB_CERTS_DIR explicitly set to empty - forcing UNSECURE mode" - FORCE_UNSECURE=1 -elif [ ! -d "$WEIGHTSLAB_CERTS_DIR" ]; then - # WEIGHTSLAB_CERTS_DIR doesn't exist, try converting Windows path to Unix-style - CONVERTED_PATH="" - if echo "$WEIGHTSLAB_CERTS_DIR" | grep -q '\\'; then - # Path contains backslashes - likely Windows path, convert to Unix-style (for Git Bash) - # Convert C:\path\to\dir -> /c/path/to/dir - CONVERTED_PATH=$(echo "$WEIGHTSLAB_CERTS_DIR" | sed 's/^\([A-Za-z]\):\\/\/\L\1\//; s/\\/\//g') - echo "Detected Windows path, converting to Unix-style: $CONVERTED_PATH" - - if [ -d "$CONVERTED_PATH" ]; then - echo "Found converted path at '$CONVERTED_PATH'" - WEIGHTSLAB_CERTS_DIR="$CONVERTED_PATH" - else - echo "Converted path not found at '$CONVERTED_PATH'" - # Continue with next fallback - WEIGHTSLAB_CERTS_DIR="" - fi - else - # Not a Windows path, try default ~/.weightslab-certs - DEFAULT_CERTS_DIR="$HOME/.weightslab-certs" - if [ -d "$DEFAULT_CERTS_DIR" ]; then - echo "WEIGHTSLAB_CERTS_DIR not found at '$WEIGHTSLAB_CERTS_DIR', using default: $DEFAULT_CERTS_DIR" - WEIGHTSLAB_CERTS_DIR="$DEFAULT_CERTS_DIR" - else - echo "Default certs directory not found ($DEFAULT_CERTS_DIR), continuing without certs" - WEIGHTSLAB_CERTS_DIR="" - fi - fi -fi - -TOKEN_FILE="${WEIGHTSLAB_CERTS_DIR}/.grpc_auth_token" -echo "Looking at Weightslab Certs dir directory at: ${TOKEN_FILE}" -if [ -n "$WEIGHTSLAB_CERTS_DIR" ]; then - ls -la "$WEIGHTSLAB_CERTS_DIR" 2>/dev/null || echo "WEIGHTSLAB_CERTS_DIR path does not exist or is empty" -else - echo "WEIGHTSLAB_CERTS_DIR is empty, skipping cert check" -fi - -# --------------------------------------------------------------------------- -# Single source of truth: TLS + gRPC auth are derived SOLELY from the presence -# of cert files in WEIGHTSLAB_CERTS_DIR. Any inherited VITE_*/ENVOY_* values are -# intentionally ignored and recomputed here, so a stale or pre-set env var can -# never force HTTPS without certs (which would crash Envoy on an empty mount) or -# vice versa. `--unsecure` (or an empty WEIGHTSLAB_CERTS_DIR) forces everything -# off. -# --------------------------------------------------------------------------- -if [ "$FORCE_UNSECURE" = "1" ] || [ -z "$WEIGHTSLAB_CERTS_DIR" ]; then - echo "Unsecured mode: HTTP, no auth (no certs directory)" - VITE_DEV_SERVER_HTTPS=0 - VITE_SERVER_PROTOCOL=http - ENVOY_UPSTREAM_TLS=off - ENVOY_DOWNSTREAM_TLS=off - VITE_WL_ENABLE_GRPC_AUTH_TOKEN=0 - VITE_GRPC_AUTH_TOKEN="" -else - # Downstream HTTPS (browser <-> Envoy): needs the Envoy server cert + key. - if [ -f "$WEIGHTSLAB_CERTS_DIR/envoy-server.crt" ] && [ -f "$WEIGHTSLAB_CERTS_DIR/envoy-server.key" ]; then - echo "Envoy server cert found in $WEIGHTSLAB_CERTS_DIR - enabling HTTPS" - VITE_DEV_SERVER_HTTPS=1 - VITE_SERVER_PROTOCOL=https - ENVOY_DOWNSTREAM_TLS=on - else - echo "No Envoy server cert in $WEIGHTSLAB_CERTS_DIR - HTTP (no downstream TLS)" - VITE_DEV_SERVER_HTTPS=0 - VITE_SERVER_PROTOCOL=http - ENVOY_DOWNSTREAM_TLS=off - fi - - # Upstream mTLS (Envoy <-> backend gRPC): needs the Envoy client cert + CA. - if [ -f "$WEIGHTSLAB_CERTS_DIR/envoy-client.crt" ] && [ -f "$WEIGHTSLAB_CERTS_DIR/envoy-client.key" ] && [ -f "$WEIGHTSLAB_CERTS_DIR/ca.crt" ]; then - echo "Envoy client cert + CA found - enabling upstream TLS" - ENVOY_UPSTREAM_TLS=on - else - echo "No Envoy client cert/CA - upstream plaintext" - ENVOY_UPSTREAM_TLS=off - fi - - # gRPC auth token. - if [ -f "$TOKEN_FILE" ]; then - echo "gRPC token found - enabling auth" - VITE_WL_ENABLE_GRPC_AUTH_TOKEN=1 - VITE_GRPC_AUTH_TOKEN=$(cat "$TOKEN_FILE") - else - echo "gRPC token not found - auth disabled" - VITE_WL_ENABLE_GRPC_AUTH_TOKEN=0 - VITE_GRPC_AUTH_TOKEN="" - fi -fi - -# Export all derived variables for docker compose -export VITE_DEV_SERVER_HTTPS -export VITE_SERVER_PROTOCOL -export ENVOY_UPSTREAM_TLS -export ENVOY_DOWNSTREAM_TLS -export VITE_WL_ENABLE_GRPC_AUTH_TOKEN -export VITE_GRPC_AUTH_TOKEN - -# Get weightslab root from environment variable or derive from script location -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# The compose stack lives one level up from this script (docker/docker/utils/ -> -# docker/docker/), where docker-compose.yml and the generated .env belong. Always -# derive it from the script location: it is correct both in-repo and pip-installed, -# and never drifts when the package layout changes. (A stale hardcoded -# "$WEIGHTSLAB_ROOT/weightslab/ui/docker" here wrote .env to a non-existent dir, so -# docker compose fell back to its defaults — HTTPS on but gRPC auth off.) -if [ -n "$WEIGHTSLAB_ROOT" ]; then - echo "WEIGHTSLAB_ROOT from environment: $WEIGHTSLAB_ROOT (compose dir derived from script location)" -fi -DOCKER_DIR="$(dirname "$SCRIPT_DIR")" - -ENV_FILE="$DOCKER_DIR/.env" -echo "SCRIPT_DIR: $SCRIPT_DIR" -echo "DOCKER_DIR: $DOCKER_DIR" - -# Write environment variables to .env file for docker compose -echo "Writing environment variables to .env..." -# Note: .env must be in the docker directory where docker-compose.yml is located - -# Convert WEIGHTSLAB_CERTS_DIR to absolute path for docker-compose compatibility -if [ -n "$WEIGHTSLAB_CERTS_DIR" ] && [ ! -z "$WEIGHTSLAB_CERTS_DIR" ]; then - # Resolve to absolute path - WEIGHTSLAB_CERTS_DIR_ABSOLUTE="$(cd "$WEIGHTSLAB_CERTS_DIR" 2>/dev/null && pwd)" || WEIGHTSLAB_CERTS_DIR_ABSOLUTE="$WEIGHTSLAB_CERTS_DIR" -fi - -# Path written into .env for the docker compose bind mount. Prefer the -# host-native path provided by the Python launcher (e.g. C:/Users/... on -# Windows/Docker Desktop): the Unix path above (/mnt/c/...) is only valid for -# this script's own file checks, NOT for a Docker Desktop bind mount, which -# silently mounts an empty dir for /mnt paths and crashes Envoy on missing certs. -CERTS_DIR_FOR_ENV="${WEIGHTSLAB_CERTS_DIR_HOST:-$WEIGHTSLAB_CERTS_DIR_ABSOLUTE}" - -if [ -z "$WEIGHTSLAB_CERTS_DIR" ]; then - # Unsecured mode: explicitly disable all TLS and auth variables - cat > "$ENV_FILE" << EOF -VITE_DEV_SERVER_HTTPS=0 -VITE_SERVER_PROTOCOL=http -VITE_WL_ENABLE_GRPC_AUTH_TOKEN=0 -VITE_GRPC_AUTH_TOKEN= -WEIGHTSLAB_CERTS_DIR= -ENVOY_UPSTREAM_TLS=off -ENVOY_DOWNSTREAM_TLS=off -WS_SERVER_PROTOCOL=http -GRPC_BACKEND_PORT=${GRPC_BACKEND_PORT:-50051} -BB_THUMB_RENDER=${BB_THUMB_RENDER:-10} -BB_MODAL_RENDER=${BB_MODAL_RENDER:-100} -WS_HISTOGRAM_MAX_BINS=${WS_HISTOGRAM_MAX_BINS:-512} -VITE_HISTOGRAM_MAX_BINS=${WS_HISTOGRAM_MAX_BINS:-512} -GRID_WINDOW_SIZE=${GRID_WINDOW_SIZE:-6} -VITE_GRID_WINDOW_SIZE=${GRID_WINDOW_SIZE:-6} -GRID_CACHE_MAX_MB=${GRID_CACHE_MAX_MB:-128} -VITE_WS_GRID_CACHE_MAX_MB=${GRID_CACHE_MAX_MB:-128} -MODAL_CACHE_MAX_MB=${MODAL_CACHE_MAX_MB:-64} -VITE_WS_MODAL_CACHE_MAX_MB=${MODAL_CACHE_MAX_MB:-64} -PC_MAX_POINTS=${PC_MAX_POINTS:-} -VITE_WL_PC_MAX_POINTS=${PC_MAX_POINTS:-} -DISABLE_GPU_RENDERING=${DISABLE_GPU_RENDERING:-0} -VITE_WL_DISABLE_GPU_RENDERING=${DISABLE_GPU_RENDERING:-0} -ENABLE_PLOTS=${ENABLE_PLOTS:-1} -ENABLE_DATA_EXPLORATION=${ENABLE_DATA_EXPLORATION:-1} -ENABLE_HYPERPARAMETERS_OPTIMIZATION=${ENABLE_HYPERPARAMETERS_OPTIMIZATION:-1} -ENABLE_AGENT=${ENABLE_AGENT:-1} -EOF -else - # Secured mode: include certs - cat > "$ENV_FILE" << EOF -VITE_DEV_SERVER_HTTPS=$VITE_DEV_SERVER_HTTPS -VITE_SERVER_PROTOCOL=$VITE_SERVER_PROTOCOL -VITE_WL_ENABLE_GRPC_AUTH_TOKEN=$VITE_WL_ENABLE_GRPC_AUTH_TOKEN -VITE_GRPC_AUTH_TOKEN=$VITE_GRPC_AUTH_TOKEN -WEIGHTSLAB_CERTS_DIR=$CERTS_DIR_FOR_ENV -ENVOY_UPSTREAM_TLS=$ENVOY_UPSTREAM_TLS -ENVOY_DOWNSTREAM_TLS=$ENVOY_DOWNSTREAM_TLS -WS_SERVER_PROTOCOL=$VITE_SERVER_PROTOCOL -GRPC_BACKEND_PORT=${GRPC_BACKEND_PORT:-50051} -BB_THUMB_RENDER=${BB_THUMB_RENDER:-10} -BB_MODAL_RENDER=${BB_MODAL_RENDER:-100} -WS_HISTOGRAM_MAX_BINS=${WS_HISTOGRAM_MAX_BINS:-512} -VITE_HISTOGRAM_MAX_BINS=${WS_HISTOGRAM_MAX_BINS:-512} -GRID_WINDOW_SIZE=${GRID_WINDOW_SIZE:-6} -VITE_GRID_WINDOW_SIZE=${GRID_WINDOW_SIZE:-6} -GRID_CACHE_MAX_MB=${GRID_CACHE_MAX_MB:-128} -VITE_WS_GRID_CACHE_MAX_MB=${GRID_CACHE_MAX_MB:-128} -MODAL_CACHE_MAX_MB=${MODAL_CACHE_MAX_MB:-64} -VITE_WS_MODAL_CACHE_MAX_MB=${MODAL_CACHE_MAX_MB:-64} -PC_MAX_POINTS=${PC_MAX_POINTS:-} -VITE_WL_PC_MAX_POINTS=${PC_MAX_POINTS:-} -DISABLE_GPU_RENDERING=${DISABLE_GPU_RENDERING:-0} -VITE_WL_DISABLE_GPU_RENDERING=${DISABLE_GPU_RENDERING:-0} -ENABLE_PLOTS=${ENABLE_PLOTS:-1} -ENABLE_DATA_EXPLORATION=${ENABLE_DATA_EXPLORATION:-1} -ENABLE_HYPERPARAMETERS_OPTIMIZATION=${ENABLE_HYPERPARAMETERS_OPTIMIZATION:-1} -ENABLE_AGENT=${ENABLE_AGENT:-1} -EOF -fi -echo ".env file written to $ENV_FILE" -cat "$ENV_FILE" - -# Convert WEIGHTSLAB_CERTS_DIR to absolute path for docker-compose compatibility -if [ -n "$WEIGHTSLAB_CERTS_DIR" ] && [ ! -z "$WEIGHTSLAB_CERTS_DIR" ]; then - # Resolve to absolute path - WEIGHTSLAB_CERTS_DIR_ABSOLUTE="$(cd "$WEIGHTSLAB_CERTS_DIR" 2>/dev/null && pwd)" || WEIGHTSLAB_CERTS_DIR_ABSOLUTE="$WEIGHTSLAB_CERTS_DIR" -fi - -# All cert/token file checks above used the Unix path (e.g. /mnt/c/...) so they -# resolve under this shell. But docker compose gives an exported shell env var -# precedence over the .env file, and Docker Desktop cannot bind-mount a -# /mnt-style source (it silently mounts an empty dir, crashing Envoy on missing -# certs). Re-export the host-native path so the compose bind mount matches .env. -if [ -n "$CERTS_DIR_FOR_ENV" ]; then - export WEIGHTSLAB_CERTS_DIR="$CERTS_DIR_FOR_ENV" - echo "Exporting host-native WEIGHTSLAB_CERTS_DIR for docker compose: $WEIGHTSLAB_CERTS_DIR" -fi - -# Check if image already exists -IMAGE_NAME="graybx/weightslab" -# if docker image inspect "$IMAGE_NAME" > /dev/null 2>&1; then -# echo "Image '$IMAGE_NAME' already exists - skipping build" -# SKIP_BUILD=true -# else -# echo "Image '$IMAGE_NAME' not found - will build" -# SKIP_BUILD=false -# fi -SKIP_BUILD=false - -# When invoked from the Python launcher on a host where docker runs outside this -# shell (e.g. Windows: docker lives on the host, not in WSL), the launcher does -# `docker compose pull/up` itself. In that case this script only needs to write -# the .env above — skip all docker operations here to avoid noisy failures. -if [ -n "$WEIGHTSLAB_SKIP_DOCKER_OPS" ] && [ "$WEIGHTSLAB_SKIP_DOCKER_OPS" != "0" ]; then - echo "Skipping docker build/deploy in shell (handled by the launcher)." - exit 0 -fi - -# Detect the Docker Compose CLI: prefer v2 (the `docker compose` plugin), fall -# back to the legacy v1 standalone binary (`docker-compose`). This lets the -# deploy work whether the user has Compose v2 or v1 installed. -if docker compose version >/dev/null 2>&1; then - DOCKER_COMPOSE="docker compose" -elif command -v docker-compose >/dev/null 2>&1; then - DOCKER_COMPOSE="docker-compose" -else - echo "Error: Docker Compose not found." >&2 - echo " Install Compose v2 (the 'docker compose' plugin, recommended) or" >&2 - echo " the legacy v1 'docker-compose' binary." >&2 - echo " See: https://docs.docker.com/compose/install/" >&2 - exit 1 -fi -echo "Using Docker Compose command: $DOCKER_COMPOSE" - -# Build and deploy -if [ "$DEV" = "true" ]; then - echo "Skipped dev build (image already exists)" - # Deploy (docker compose automatically reads .env) - echo "Deploying containers..." - $DOCKER_COMPOSE -f $DOCKER_DIR/docker-compose.yml -f $DOCKER_DIR/docker-compose.dev.yml down - $DOCKER_COMPOSE -f $DOCKER_DIR/docker-compose.yml -f $DOCKER_DIR/docker-compose.dev.yml up -d --force-recreate - - echo "Deployment to development complete!" -else - if [ "$SKIP_BUILD" = "false" ]; then - echo "Building production image (single image, configuration at runtime)..." - # Build with defaults - configuration happens at runtime via .env - $DOCKER_COMPOSE -f $DOCKER_DIR/docker-compose.yml build --no-cache - - BUILD_STATUS=$? - if [ $BUILD_STATUS -ne 0 ]; then - echo "Production build failed with status $BUILD_STATUS" - exit $BUILD_STATUS - fi - - echo "Production build complete!" - else - echo "Skipped production build (image already exists)" - fi - - # Deploy (docker compose automatically reads .env) - echo "Deploying containers..." - echo "Stopping existing containers..." - $DOCKER_COMPOSE -f $DOCKER_DIR/docker-compose.yml down - - echo "Starting containers..." - $DOCKER_COMPOSE -f $DOCKER_DIR/docker-compose.yml up -d --force-recreate - - UP_STATUS=$? - if [ $UP_STATUS -ne 0 ]; then - echo "Container startup failed with status $UP_STATUS" - echo "Checking container logs..." - $DOCKER_COMPOSE -f $DOCKER_DIR/docker-compose.yml logs --tail=50 || true - exit $UP_STATUS - fi - - echo "Deployment to production complete!" - echo "Running containers:" - $DOCKER_COMPOSE -f $DOCKER_DIR/docker-compose.yml ps || true -fi diff --git a/weightslab/docker/docker/utils/test-deployment.sh b/weightslab/docker/docker/utils/test-deployment.sh deleted file mode 100644 index 11c22ef3..00000000 --- a/weightslab/docker/docker/utils/test-deployment.sh +++ /dev/null @@ -1,437 +0,0 @@ -#!/bin/bash -# Comprehensive deployment test for weights_studio frontend + backend -# Tests communication between frontend and Python backend (ws-segmentation example) -# Runs both unsecured (HTTP) and secured (HTTPS) test scenarios - -# set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Configuration -CURRENT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" -DOCKER_DIR="$CURRENT_DIR/docker" -UTILS_DIR="$DOCKER_DIR/utils" -EXAMPLE_DIR="$CURRENT_DIR/tests/playwright/examples/ws-segmentation" -GRPC_PORT=50051 -ENVOY_PORT=8080 -FRONTEND_PORT=5173 -HEALTH_CHECK_TIMEOUT=30 -COMMUNICATION_TEST_TIMEOUT=20 - -# Global test results -TESTS_PASSED=0 -TESTS_FAILED=0 - -# ============================================================================ -# Utility Functions -# ============================================================================ - -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[✓ PASS]${NC} $1" - ((TESTS_PASSED++)) -} - -log_error() { - echo -e "${RED}[✗ FAIL]${NC} $1" - ((TESTS_FAILED++)) -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -print_section() { - echo "" - echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}" - echo -e "${BLUE}║${NC} $1" - echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}" - echo "" -} - -# Clean up docker resources -cleanup() { - log_info "Cleaning up Docker resources..." - cd "$DOCKER_DIR" - docker compose -f docker-compose.yml -f docker-compose.prod.yml down --remove-orphans 2>/dev/null || true - # Force remove any lingering containers - docker ps -a | grep weights_studio | awk '{print $1}' | xargs -r docker rm -f 2>/dev/null || true - # Remove network - docker network rm docker_weights_studio_network 2>/dev/null || true - sleep 3 -} - -# Wait for service to be ready -wait_for_service() { - local service_name=$1 - local port=$2 - local timeout=$3 - local protocol=${4:-http} - - log_info "Waiting for $service_name on $protocol://localhost:$port..." - - local start_time=$(date +%s) - while true; do - local current_time=$(date +%s) - local elapsed=$((current_time - start_time)) - - if [ $elapsed -gt $timeout ]; then - log_error "$service_name failed to start after ${timeout}s" - return 1 - fi - - if [ "$protocol" = "https" ]; then - if curl -sk -o /dev/null -w "%{http_code}" "https://localhost:$port/health" 2>/dev/null | grep -q "200"; then - log_success "$service_name is ready" - return 0 - fi - else - if curl -s -o /dev/null -w "%{http_code}" "http://localhost:$port/health" 2>/dev/null | grep -q "200"; then - log_success "$service_name is ready" - return 0 - fi - fi - - sleep 2 - done -} - -# Check if gRPC backend is responding -check_grpc_backend() { - local timeout=$1 - log_info "Checking gRPC backend on localhost:$GRPC_PORT..." - - local start_time=$(date +%s) - while true; do - local current_time=$(date +%s) - local elapsed=$((current_time - start_time)) - - if [ $elapsed -gt $timeout ]; then - log_error "gRPC backend not responding after ${timeout}s" - return 1 - fi - - # Try to connect using nc (netcat) - if nc -z localhost $GRPC_PORT 2>/dev/null; then - log_success "gRPC backend is listening on port $GRPC_PORT" - return 0 - fi - - sleep 1 - done -} - -# Check frontend-backend communication -test_frontend_backend_communication() { - local protocol=$1 - local test_name=$2 - local port=${3:-$FRONTEND_PORT} - - log_info "Testing frontend -> backend communication ($test_name)..." - - # The frontend makes requests to /api which proxies to envoy - # Envoy proxies to localhost:50051 (the gRPC backend) - - # Check if frontend can resolve the backend - local health_url="$protocol://localhost:$port/health" - - local http_code=$(curl -sk -o /dev/null -w "%{http_code}" "$health_url" 2>/dev/null || echo "000") - - if [ "$http_code" = "200" ]; then - log_success "Frontend is responding ($test_name)" - else - log_error "Frontend health check failed with HTTP $http_code ($test_name)" - return 1 - fi - - return 0 -} - -# Run backend example in background and monitor -run_backend_example() { - local test_scenario=$1 - - if [ ! -f "$EXAMPLE_DIR/main.py" ]; then - log_error "Python example not found at $EXAMPLE_DIR/main.py" - return 1 - fi - - log_info "Starting backend example: $test_scenario..." - - cd "$EXAMPLE_DIR" - - # Start Python backend in background - timeout 60s python main.py > "/tmp/ws_backend_${test_scenario}.log" 2>&1 & - local backend_pid=$! 2>/dev/null || true - - log_info "Backend process started (PID: $backend_pid)" - - # Wait a bit for backend to initialize - sleep 3 - - # Check if process is still running - if ! kill -0 $backend_pid 2>/dev/null; then - log_error "Backend process exited prematurely" - cat "/tmp/ws_backend_${test_scenario}.log" - return 1 - fi - - log_success "Backend example is running" - - # Check gRPC port - if check_grpc_backend $COMMUNICATION_TEST_TIMEOUT; then - log_success "Backend gRPC service is reachable" - else - log_error "Backend gRPC service is not reachable" - kill $backend_pid 2>/dev/null || true - return 1 - fi - - return 0 -} - -# Temporarily hide/show certs and tokens for testing -hide_certs() { - local certs_dir=$1 - if [ -d "$certs_dir" ]; then - mv "$certs_dir" "${certs_dir}.bak" - mkdir -p "$certs_dir" - fi -} - -restore_certs() { - local certs_dir=$1 - if [ -d "${certs_dir}.bak" ]; then - rm -rf "$certs_dir" - mv "${certs_dir}.bak" "$certs_dir" - fi -} - -hide_grpc_token() { - local token_file=$1 - if [ -f "$token_file" ]; then - mv "$token_file" "${token_file}.bak" - fi -} - -restore_grpc_token() { - local token_file=$1 - if [ -f "${token_file}.bak" ]; then - mv "${token_file}.bak" "$token_file" - fi -} - -# Run test scenario -run_test_scenario() { - local scenario_name=$1 - local secure=$2 - local use_certs=${3:-true} - local use_auth=${4:-false} - - # Create a safe filename from scenario name (remove spaces, colons, etc) - local scenario_safe=$(echo "$scenario_name" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/-\+/-/g' | sed 's/-$//') - - print_section "$scenario_name" - - # Cleanup previous deployment - cleanup - - # Setup certs and tokens for this scenario - local certs_dir="$DOCKER_DIR/.weightslab-certs-test" - log_info "certs_dir: $certs_dir" - local token_file="$certs_dir/.grpc_auth_token" - local certs_hidden=false - local token_hidden=false - - if [ "$use_certs" = "false" ] && [ -d "$certs_dir" ]; then - log_info "Temporarily hiding TLS certificates for this test..." - hide_certs "$certs_dir" - certs_hidden=true - fi - - if [ "$use_auth" = "false" ] && [ -f "$token_file" ]; then - log_info "Temporarily hiding gRPC auth token for this test..." - hide_grpc_token "$token_file" - token_hidden=true - fi - - # Set environment for deployment - if [ "$secure" = "true" ]; then - if [ "$use_certs" = "true" ] && [ "$use_auth" = "true" ]; then - log_info "Secure mode: HTTPS with TLS + gRPC auth enabled" - elif [ "$use_certs" = "true" ]; then - log_info "Secure mode: HTTPS with TLS (no gRPC auth)" - elif [ "$use_auth" = "true" ]; then - log_info "Secure mode: HTTP with gRPC auth only (no TLS)" - fi - export E2E_MANAGED_STACK=1 - else - log_info "Unsecure mode: HTTP without TLS or gRPC auth" - export E2E_MANAGED_STACK=1 - fi - - # Build and deploy frontend - cd "$UTILS_DIR" - - if [ "$secure" = "true" ] && [ "$use_certs" = "true" ]; then - log_info "Building and deploying with HTTPS support..." - WEIGHTSLAB_CERTS_DIR="$certs_dir" bash build-and-deploy.sh > "/tmp/build_deploy_${scenario_safe}.log" 2>&1 || { - log_error "Build and deploy failed" - cat "/tmp/build_deploy_${scenario_safe}.log" - [ "$certs_hidden" = true ] && restore_certs "$certs_dir" - [ "$token_hidden" = true ] && restore_grpc_token "$token_file" - return 1 - } - else - log_info "Building and deploying with HTTP (no TLS certs)..." - WEIGHTSLAB_CERTS_DIR="$certs_dir" bash build-and-deploy.sh --unsecure > "/tmp/build_deploy_${scenario_safe}.log" 2>&1 || { - log_error "Build and deploy failed" - cat "/tmp/build_deploy_${scenario_safe}.log" - [ "$certs_hidden" = true ] && restore_certs "$certs_dir" - [ "$token_hidden" = true ] && restore_grpc_token "$token_file" - return 1 - } - fi - - log_success "Frontend deployment successful" - - # Wait for services to be ready - local protocol="http" - local frontend_port=$FRONTEND_PORT - if [ "$secure" = "true" ] && [ "$use_certs" = "true" ]; then - protocol="https" - fi - - # Check frontend (don't check envoy since it depends on upstream backend) - if ! wait_for_service "Frontend (nginx)" "$frontend_port" "$HEALTH_CHECK_TIMEOUT" "$protocol"; then - log_error "Frontend failed to start" - [ "$certs_hidden" = true ] && restore_certs "$certs_dir" - [ "$token_hidden" = true ] && restore_grpc_token "$token_file" - return 1 - fi - - # Test frontend-backend communication setup - if test_frontend_backend_communication "$protocol" "$scenario_safe" "$frontend_port"; then - log_success "Frontend-backend communication setup OK" - else - log_error "Frontend-backend communication setup failed" - [ "$certs_hidden" = true ] && restore_certs "$certs_dir" - [ "$token_hidden" = true ] && restore_grpc_token "$token_file" - return 1 - fi - - # Restore certs and tokens - if [ "$certs_hidden" = true ]; then - log_info "Restoring TLS certificates..." - restore_certs "$certs_dir" - fi - if [ "$token_hidden" = true ]; then - log_info "Restoring gRPC auth token..." - restore_grpc_token "$token_file" - fi - - return 0 -} - -# ============================================================================ -# Main Test Execution -# ============================================================================ - -main() { - print_section "WEIGHTS STUDIO FRONTEND-BACKEND DEPLOYMENT TEST" - - log_info "Test Configuration:" - log_info " Weights Studio Dir: $CURRENT_DIR" - log_info " Docker Dir: $DOCKER_DIR" - log_info " Example Dir: $EXAMPLE_DIR" - log_info " Envoy Port: $ENVOY_PORT" - log_info " Frontend Port: $FRONTEND_PORT" - log_info " gRPC Backend Port: $GRPC_PORT" - - # Test A: Unsecured Environment (HTTP) - if run_test_scenario "TEST A: UNSECURED ENVIRONMENT (HTTP)" "false" "false" "false"; then - log_success "TEST A PASSED: Frontend and backend communicate over HTTP" - else - log_error "TEST A FAILED: Frontend-backend communication issue in HTTP mode" - fi - - # Cleanup between tests - cleanup - sleep 5 - - # Test B: Secured Environment (HTTPS with TLS + gRPC Auth) - # Note: This requires TLS certificates AND gRPC auth token to be present - if [ -d "$DOCKER_DIR/.weightslab-certs-test" ] && [ -f "$DOCKER_DIR/.weightslab-certs-test/envoy-server.crt" ]; then - if [ -f "$DOCKER_DIR/.weightslab-certs-test/.grpc_auth_token" ]; then - if run_test_scenario "TEST B: SECURED ENVIRONMENT (HTTPS + TLS + gRPC AUTH)" "true" "true" "true"; then - log_success "TEST B PASSED: Frontend and backend communicate over HTTPS with full security" - else - log_error "TEST B FAILED: Frontend-backend communication issue in HTTPS+Auth mode" - fi - else - log_warn "TEST B SKIPPED: gRPC auth token not found" - log_warn " Token should be at $DOCKER_DIR/.weightslab-certs-test/.grpc_auth_token" - fi - - # Cleanup between tests - cleanup - sleep 5 - - # Test C: TLS Certificates Only (no gRPC auth) - if run_test_scenario "TEST C: SECURED ENVIRONMENT (HTTPS + TLS ONLY, NO GRPC AUTH)" "true" "true" "false"; then - log_success "TEST C PASSED: Frontend and backend communicate over HTTPS (TLS only, no gRPC auth)" - else - log_error "TEST C FAILED: Frontend-backend communication issue in TLS-only mode" - fi - - # Cleanup between tests - cleanup - sleep 5 - - # Test D: gRPC Auth Only (no TLS certificates) - if [ -f "$DOCKER_DIR/.weightslab-certs-test/.grpc_auth_token" ]; then - if run_test_scenario "TEST D: SECURED ENVIRONMENT (HTTP + gRPC AUTH ONLY, NO TLS)" "true" "false" "true"; then - log_success "TEST D PASSED: Frontend and backend communicate over HTTP with gRPC auth only" - else - log_error "TEST D FAILED: Frontend-backend communication issue in gRPC-auth-only mode" - fi - else - log_warn "TEST D SKIPPED: gRPC auth token not found" - log_warn " Token should be at $DOCKER_DIR/.weightslab-certs-test/.grpc_auth_token" - fi - else - log_warn "TESTS B, C, D SKIPPED: TLS certificates not found at $DOCKER_DIR/.weightslab-certs-test" - log_warn " To run secured tests, copy certs to:" - log_warn " $DOCKER_DIR/.weightslab-certs-test" - fi - - # Final cleanup - cleanup - - # Print summary - print_section "TEST SUMMARY" - echo -e "Tests Passed: ${GREEN}$TESTS_PASSED${NC}" - echo -e "Tests Failed: ${RED}$TESTS_FAILED${NC}" - - if [ $TESTS_FAILED -eq 0 ]; then - log_success "ALL TESTS PASSED! ✨" - return 0 - else - log_error "SOME TESTS FAILED ❌" - return 1 - fi -} - -# Handle script interruption -trap cleanup EXIT - -# Run main -main "$@" diff --git a/weightslab/docker/envoy/envoy.downstream_plaintext.yaml b/weightslab/docker/envoy/envoy.downstream_plaintext.yaml deleted file mode 100644 index d96b8f44..00000000 --- a/weightslab/docker/envoy/envoy.downstream_plaintext.yaml +++ /dev/null @@ -1,100 +0,0 @@ -static_resources: - listeners: - - name: listener_0 - per_connection_buffer_limit_bytes: 268435456 # 256 MB — matches GRPC_MAX_MESSAGE_BYTES - address: - socket_address: - address: 0.0.0.0 - port_value: 8080 - filter_chains: - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: AUTO - stat_prefix: ingress_http - route_config: - name: local_route - virtual_hosts: - - name: local_service - domains: ["*"] - routes: - - match: - prefix: "/" - route: - cluster: grpc_service - timeout: 300s - retry_policy: - retry_on: connect-failure,refused-stream,reset - num_retries: 1 - per_try_timeout: 60s - max_stream_duration: - grpc_timeout_header_max: 300s - cors: - allow_origin_string_match: - - exact: "http://localhost" - - exact: "http://127.0.0.1" - - exact: "http://localhost:5173" - - exact: "http://127.0.0.1:5173" - - exact: "https://localhost" - - exact: "https://127.0.0.1" - - exact: "https://localhost:5173" - - exact: "https://127.0.0.1:5173" - allow_methods: GET, PUT, DELETE, POST, OPTIONS - allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout - max_age: "1728000" - expose_headers: custom-header-1,grpc-status,grpc-message - http_filters: - - name: envoy.filters.http.grpc_web - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb - - name: envoy.filters.http.cors - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - - clusters: - - name: grpc_service - connect_timeout: 1s - type: STRICT_DNS - lb_policy: ROUND_ROBIN - dns_lookup_family: V4_ONLY - http2_protocol_options: {} - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 100 - max_pending_requests: 100 - max_requests: 200 - max_retries: 3 - load_assignment: - cluster_name: grpc_service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: grpc-backend - port_value: __GRPC_BACKEND_PORT__ - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext - common_tls_context: - tls_certificates: - - certificate_chain: - filename: /etc/envoy/certs/envoy-client.crt - private_key: - filename: /etc/envoy/certs/envoy-client.key - validation_context: - trusted_ca: - filename: /etc/envoy/certs/ca.crt - -admin: - access_log_path: /tmp/admin_access.log - address: - socket_address: - address: 127.0.0.1 - port_value: 9901 diff --git a/weightslab/docker/envoy/envoy.downstream_upstream_plaintext.yaml b/weightslab/docker/envoy/envoy.downstream_upstream_plaintext.yaml deleted file mode 100644 index 3bee23b9..00000000 --- a/weightslab/docker/envoy/envoy.downstream_upstream_plaintext.yaml +++ /dev/null @@ -1,87 +0,0 @@ -static_resources: - listeners: - - name: listener_0 - per_connection_buffer_limit_bytes: 268435456 # 256 MB — matches GRPC_MAX_MESSAGE_BYTES - address: - socket_address: - address: 0.0.0.0 - port_value: 8080 - filter_chains: - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: AUTO - stat_prefix: ingress_http - route_config: - name: local_route - virtual_hosts: - - name: local_service - domains: ["*"] - routes: - - match: - prefix: "/" - route: - cluster: grpc_service - timeout: 300s - retry_policy: - retry_on: connect-failure,refused-stream,reset - num_retries: 1 - per_try_timeout: 60s - max_stream_duration: - grpc_timeout_header_max: 300s - cors: - allow_origin_string_match: - - exact: "http://localhost" - - exact: "http://127.0.0.1" - - exact: "http://localhost:5173" - - exact: "http://127.0.0.1:5173" - - exact: "https://localhost" - - exact: "https://127.0.0.1" - - exact: "https://localhost:5173" - - exact: "https://127.0.0.1:5173" - allow_methods: GET, PUT, DELETE, POST, OPTIONS - allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout - max_age: "1728000" - expose_headers: custom-header-1,grpc-status,grpc-message - http_filters: - - name: envoy.filters.http.grpc_web - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb - - name: envoy.filters.http.cors - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - - clusters: - - name: grpc_service - connect_timeout: 1s - type: STRICT_DNS - lb_policy: ROUND_ROBIN - dns_lookup_family: V4_ONLY - http2_protocol_options: {} - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 100 - max_pending_requests: 100 - max_requests: 200 - max_retries: 3 - load_assignment: - cluster_name: grpc_service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: grpc-backend - port_value: __GRPC_BACKEND_PORT__ - -admin: - access_log_path: /tmp/admin_access.log - address: - socket_address: - address: 127.0.0.1 - port_value: 9901 diff --git a/weightslab/docker/envoy/envoy.upstream_plaintext.yaml b/weightslab/docker/envoy/envoy.upstream_plaintext.yaml deleted file mode 100644 index 38f65e28..00000000 --- a/weightslab/docker/envoy/envoy.upstream_plaintext.yaml +++ /dev/null @@ -1,97 +0,0 @@ -static_resources: - listeners: - - name: listener_0 - per_connection_buffer_limit_bytes: 268435456 # 256 MB — matches GRPC_MAX_MESSAGE_BYTES - address: - socket_address: - address: 0.0.0.0 - port_value: 8080 - filter_chains: - - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext - common_tls_context: - tls_certificates: - - certificate_chain: - filename: /etc/envoy/certs/envoy-server.crt - private_key: - filename: /etc/envoy/certs/envoy-server.key - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: AUTO - stat_prefix: ingress_http - route_config: - name: local_route - virtual_hosts: - - name: local_service - domains: ["*"] - routes: - - match: - prefix: "/" - route: - cluster: grpc_service - timeout: 300s - retry_policy: - retry_on: connect-failure,refused-stream,reset - num_retries: 1 - per_try_timeout: 60s - max_stream_duration: - grpc_timeout_header_max: 300s - cors: - allow_origin_string_match: - - exact: "http://localhost" - - exact: "http://127.0.0.1" - - exact: "http://localhost:5173" - - exact: "http://127.0.0.1:5173" - - exact: "https://localhost" - - exact: "https://127.0.0.1" - - exact: "https://localhost:5173" - - exact: "https://127.0.0.1:5173" - allow_methods: GET, PUT, DELETE, POST, OPTIONS - allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout - max_age: "1728000" - expose_headers: custom-header-1,grpc-status,grpc-message - http_filters: - - name: envoy.filters.http.grpc_web - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb - - name: envoy.filters.http.cors - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - - clusters: - - name: grpc_service - connect_timeout: 1s - type: STRICT_DNS - lb_policy: ROUND_ROBIN - dns_lookup_family: V4_ONLY - http2_protocol_options: {} - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 100 - max_pending_requests: 100 - max_requests: 200 - max_retries: 3 - load_assignment: - cluster_name: grpc_service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: grpc-backend - port_value: __GRPC_BACKEND_PORT__ - -admin: - access_log_path: /tmp/admin_access.log - address: - socket_address: - address: 127.0.0.1 - port_value: 9901 diff --git a/weightslab/docker/envoy/envoy.yaml b/weightslab/docker/envoy/envoy.yaml deleted file mode 100644 index ed3d02fa..00000000 --- a/weightslab/docker/envoy/envoy.yaml +++ /dev/null @@ -1,110 +0,0 @@ -static_resources: - listeners: - - name: listener_0 - per_connection_buffer_limit_bytes: 268435456 # 256 MB — matches GRPC_MAX_MESSAGE_BYTES - address: - socket_address: - address: 0.0.0.0 - port_value: 8080 - filter_chains: - - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext - common_tls_context: - tls_certificates: - - certificate_chain: - filename: /etc/envoy/certs/envoy-server.crt - private_key: - filename: /etc/envoy/certs/envoy-server.key - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: AUTO - stat_prefix: ingress_http - route_config: - name: local_route - virtual_hosts: - - name: local_service - domains: ["*"] - routes: - - match: - prefix: "/" - route: - cluster: grpc_service - timeout: 300s - retry_policy: - retry_on: connect-failure,refused-stream,reset - num_retries: 1 - per_try_timeout: 60s - max_stream_duration: - grpc_timeout_header_max: 300s - cors: - allow_origin_string_match: - - exact: "http://localhost" - - exact: "http://127.0.0.1" - - exact: "http://localhost:5173" - - exact: "http://127.0.0.1:5173" - - exact: "https://localhost" - - exact: "https://127.0.0.1" - - exact: "https://localhost:5173" - - exact: "https://127.0.0.1:5173" - allow_methods: GET, PUT, DELETE, POST, OPTIONS - allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout,authorization,x-grpc-token - max_age: "1728000" - expose_headers: custom-header-1,grpc-status,grpc-message - http_filters: - - name: envoy.filters.http.grpc_web - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb - - name: envoy.filters.http.cors - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - - clusters: - - name: grpc_service - connect_timeout: 1s - type: STRICT_DNS - lb_policy: ROUND_ROBIN - dns_lookup_family: V4_ONLY - http2_protocol_options: {} - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 100 - max_pending_requests: 100 - max_requests: 200 - max_retries: 3 - load_assignment: - cluster_name: grpc_service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: host.docker.internal - port_value: __GRPC_BACKEND_PORT__ - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext - common_tls_context: - tls_certificates: - - certificate_chain: - filename: /etc/envoy/certs/envoy-client.crt - private_key: - filename: /etc/envoy/certs/envoy-client.key - validation_context: - trusted_ca: - filename: /etc/envoy/certs/ca.crt - -admin: - access_log_path: /tmp/admin_access.log - address: - socket_address: - address: 127.0.0.1 - port_value: 9901 diff --git a/weightslab/docker/nginx.conf b/weightslab/docker/nginx.conf deleted file mode 100644 index 7fdfb58b..00000000 --- a/weightslab/docker/nginx.conf +++ /dev/null @@ -1,118 +0,0 @@ -upstream backend { - server host.docker.internal:50051; -} - -upstream envoy { - server weights_studio_envoy:8080; -} - -# HTTP server - insecure mode -server { - listen 80; - server_name _; - - root /usr/share/nginx/html; - index index.html; - - # Cache static assets - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ { - expires 1y; - add_header Cache-Control "public, immutable"; - } - - # API proxy to backend through Envoy - location /api/ { - proxy_pass http://envoy/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # gRPC proxy - location ~ ^/grpc { - proxy_pass http://envoy; - proxy_http_version 1.1; - proxy_set_header Content-Type application/grpc; - proxy_set_header grpc-encoding gzip; - proxy_buffering off; - proxy_request_buffering off; - proxy_set_header Host $host; - } - - # SPA fallback - all other routes go to index.html - location / { - try_files $uri $uri/ /index.html; - } - - # Health check - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } -} - -# HTTPS server - secure mode -server { - listen 443 ssl; - server_name _; - - # SSL/TLS certificates - ssl_certificate /etc/nginx/certs/envoy-server.crt; - ssl_certificate_key /etc/nginx/certs/envoy-server.key; - - # SSL configuration - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers HIGH:!aNULL:!MD5; - ssl_prefer_server_ciphers on; - ssl_session_cache shared:SSL:10m; - ssl_session_timeout 10m; - - root /usr/share/nginx/html; - index index.html; - - # Cache static assets - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ { - expires 1y; - add_header Cache-Control "public, immutable"; - } - - # API proxy to backend through Envoy - location /api/ { - proxy_pass http://envoy/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # gRPC proxy - location ~ ^/grpc { - proxy_pass http://envoy; - proxy_http_version 1.1; - proxy_set_header Content-Type application/grpc; - proxy_set_header grpc-encoding gzip; - proxy_buffering off; - proxy_request_buffering off; - proxy_set_header Host $host; - } - - # SPA fallback - all other routes go to index.html - location / { - try_files $uri $uri/ /index.html; - } - - # Health check - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } -} diff --git a/weightslab/examples/Docker_training/README.md b/weightslab/examples/Docker_training/README.md deleted file mode 100644 index a96e7c6b..00000000 --- a/weightslab/examples/Docker_training/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Running WeightsLab from a "training docker" — three options - -Each option runs the WeightsLab UI stack + the classification training inside a -container so the example ends up reachable at , talking to -the training process end-to-end. They differ in **how the container gets a docker -daemon** to launch Envoy + frontend, and the configuration that follows. - -| | [docker_in_docker](docker_in_docker/) (A · DinD) | [siblings_docker](siblings_docker/) (B · DooD) | [siblings_self_contained](siblings_self_contained/) (C) | -|---|---|---|---| -| Docker daemon | own daemon, nested | host daemon (socket) | host daemon (socket) | -| Envoy/frontend run | nested in the trainer | siblings on the host | siblings on the host | -| Starts the UI via | `weightslab ui launch` | `weightslab ui launch` | own bind-mount-free `ui-compose.yml` | -| `--privileged` | **required** | no | no | -| gRPC `:50051` | not published | **published to host** | **published to host** | -| Browser ports 5173/8080 | re-published by trainer | published by siblings | published by siblings | -| Host bind mounts | resolve in-container — none | **path alignment** required | **none** (config via named volume + HTTP) | -| Host setup | none | `setup-host.sh` (clone to aligned path) | none | -| TLS (HTTPS) | ✅ `--certs` (+ backend TLS env) | ✅ `--certs` (aligned certs) | ✅ opt-in (certs via named volume) | -| Windows / Docker Desktop | ✅ works | ⚠️ awkward (use WSL2) | ✅ works natively | -| Best when | want isolation / Windows / TLS | Linux host, stock `ui launch` | "all from inside", no host prep, Windows | - -See each directory's `README.md` for the detailed wiring diagram and run steps. - -- **A (DinD)** — fully self-contained via a nested daemon; needs `--privileged`. -- **B (siblings)** — uses the stock `weightslab ui launch`; its bundled compose - bind-mounts the Envoy config + certs, so the host must see those files (path - alignment + `setup-host.sh`). Cleanest on a Linux host. -- **C (self-contained siblings)** — siblings *without* host bind mounts: it - delivers the Envoy config (and, for TLS, the certs) through named volumes over - the socket, so no host prep and it works natively on Windows. The trade-off: - it doesn't use `weightslab ui launch`. HTTP by default, TLS opt-in. - -> All three can run secured TLS — see each README's "🔒" section. **A and C -> expose a one-flag toggle:** `WEIGHTSLAB_TLS=1 docker compose up --build`. Note -> that for **every** option cert generation only configures Envoy + the frontend; -> the **backend** example also needs `GRPC_TLS_ENABLED=1` + `GRPC_TLS_CERT_DIR` -> (the toggle handles this), or Envoy's upstream mTLS fails. And the dev CA must -> be trusted on the host browser to avoid a self-signed warning. diff --git a/weightslab/examples/Docker_training/docker_in_docker/Dockerfile b/weightslab/examples/Docker_training/docker_in_docker/Dockerfile deleted file mode 100644 index cab3af85..00000000 --- a/weightslab/examples/Docker_training/docker_in_docker/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -# ============================================================================= -# WeightsLab "training docker" — Docker-in-Docker (DinD) variant -# ============================================================================= -# This image runs BOTH: -# 1. the WeightsLab CLI ("weightslab ui launch") which spins up the -# Envoy + Weights Studio frontend containers, and -# 2. the training process ("weightslab start example") which serves the -# in-process gRPC backend on :50051. -# -# DinD = the container runs its OWN docker daemon inside itself (requires -# --privileged). The Envoy/frontend containers are nested *inside* this -# container's daemon, so they share this container's network namespace and -# filesystem. See README.md for why that matters. -# ============================================================================= -FROM python:3.11-slim - -# --- System deps ------------------------------------------------------------- -# - docker engine (dockerd + CLI + compose plugin + containerd): installed via -# the official convenience script. We need the *daemon* here (DinD). -# - curl/ca-certificates/git: fetch the docker installer + optional dev install. -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl sudo ca-certificates git \ - && curl -fsSL https://get.docker.com | sh \ - && rm -rf /var/lib/apt/lists/* - -# --- WeightsLab -------------------------------------------------------------- -# Default: install the published package from PyPI (matches "if you didn't -# modify weightslab, use pip install"). To run your local dev branch instead: -# docker compose build \ -# --build-arg WEIGHTSLAB_SPEC="git+https://github.com/GrayboxTech/weightslab.git@dev" -ARG WEIGHTSLAB_SPEC=weightslab -RUN pip install --no-cache-dir "${WEIGHTSLAB_SPEC}" - -# gRPC backend port — must match what Envoy is told to dial (GRPC_BACKEND_PORT). -ENV GRPC_BACKEND_PORT=50051 - -# --- GPU (NVIDIA) ------------------------------------------------------------ -# Make the NVIDIA Container Toolkit inject the host driver (nvidia-smi + libs) -# into this container. `utility` => nvidia-smi works; `compute` => CUDA/torch. -# These are no-ops on a host without an NVIDIA GPU/toolkit (falls back to CPU). -# The actual GPU grant is requested in docker-compose.yml (deploy.resources). -# torch's default Linux wheel bundles the CUDA runtime, so no CUDA base image is -# needed — only the host driver (injected) is required for torch.cuda to work. -ENV NVIDIA_VISIBLE_DEVICES=all \ - NVIDIA_DRIVER_CAPABILITIES=compute,utility - -COPY entrypoint.sh /usr/local/bin/entrypoint.sh -# Strip any CR so the script runs under Linux bash even if checked out on Windows. -RUN sed -i 's/\r$//' /usr/local/bin/entrypoint.sh \ - && chmod +x /usr/local/bin/entrypoint.sh - -# OPTIONAL - Documentation only — EXPOSE does NOT publish ports. The host publishing is done -# by `ports:` in docker-compose.yml. Listed here purely to record intent: -# 5173 = Weights Studio frontend, 8080 = Envoy gRPC-web. -EXPOSE 5173 8080 - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/weightslab/examples/Docker_training/docker_in_docker/README.md b/weightslab/examples/Docker_training/docker_in_docker/README.md deleted file mode 100644 index b99004d8..00000000 --- a/weightslab/examples/Docker_training/docker_in_docker/README.md +++ /dev/null @@ -1,161 +0,0 @@ -# Training docker — Option A: Docker-in-Docker (DinD) - -Run the whole WeightsLab stack from **one self-contained training container** that -runs its own Docker daemon. Inside it we run, in order: - -1. `weightslab ui launch` → starts the **Envoy** + **Weights Studio frontend** - containers (nested inside this container's daemon), -2. `weightslab start example --cls` → runs the classification training, which - serves the in-process **gRPC backend** on `:50051`. - -The end result: open `http://localhost:5173` in your browser and the UI talks to -the training process end-to-end. - -## ⚙️ Required configuration — DinD training (weightslab + weights_studio) - -The **must-have** bits to start a DinD training where the browser reaches the -backend. Each maps to a concrete line in the files in this folder: - -| # | Requirement | Where | Why | -|---|---|---|---| -| 1 | **`privileged: true`** on the trainer | [docker-compose.yml](docker-compose.yml) | the inner `dockerd` cannot run otherwise | -| 2 | **Start the inner `dockerd` + wait for it** | [entrypoint.sh](entrypoint.sh) | nested daemon that hosts Envoy + frontend | -| 3 | **Persist `/var/lib/docker`** (named volume) | [docker-compose.yml](docker-compose.yml) | cache the Envoy + `graybx/weightslab` pulls across runs | -| 4 | **Publish `5173` + `8080`** to the host | [docker-compose.yml](docker-compose.yml) | the browser reaches the *nested* frontend + Envoy | -| 5 | **`GRPC_BACKEND_PORT=50051`** (matches Envoy) | [docker-compose.yml](docker-compose.yml) / [Dockerfile](Dockerfile) | Envoy's `grpc-backend:host-gateway` dials this port | -| 6 | **`WEIGHTSLAB_SKIP_DOCKER_OPS=1`** before `ui launch` | [entrypoint.sh](entrypoint.sh) | skip the in-container frontend rebuild; just pull the image | -| 7 | **Order: `ui launch` → `start example`** | [entrypoint.sh](entrypoint.sh) | UI stack up first; the example then serves the backend | -| — | *Not needed (unlike siblings):* publishing `:50051`, bind-mount path alignment | — | Envoy + backend share the trainer's netns + filesystem | - -### GPU — to run `nvidia-smi` / train on CUDA - -GPU access is wired straight into `docker compose up` (no extra files): - -| Requirement | Where | -|---|---| -| Host: NVIDIA driver **+ NVIDIA Container Toolkit** (`sudo nvidia-ctk runtime configure --runtime=docker`) | host setup | -| `deploy.resources.reservations.devices` (driver `nvidia`, `capabilities: [gpu]`) | [docker-compose.yml](docker-compose.yml) | -| `NVIDIA_VISIBLE_DEVICES=all` + `NVIDIA_DRIVER_CAPABILITIES=compute,utility` | [Dockerfile](Dockerfile) | - -> The `deploy:` GPU block is a **hard requirement**: on a host with no NVIDIA -> GPU/toolkit, comment it out in [docker-compose.yml](docker-compose.yml) or -> `up` will fail with *"could not select device driver nvidia"*. Only the -> **trainer** container needs the GPU — the nested Envoy/frontend don't. -> The entrypoint prints `nvidia-smi -L` at startup and degrades to CPU if no GPU -> is visible. On **Docker Desktop (Windows)** GPU passthrough goes through WSL2; -> combining it with `--privileged` DinD can be finicky — verify with -> `docker compose run --rm trainer nvidia-smi`. (A non-NVIDIA host, e.g. an AMD -> Radeon machine, will not expose `nvidia-smi` regardless of this config.) - -## How the wiring works (and why DinD is simple) - -``` -host browser - → localhost:5173 ── (re-published) ──► trainer container ──► inner frontend :5173 - → localhost:8080 ── (re-published) ──► trainer container ──► inner Envoy :8080 - │ grpc-backend:host-gateway - ▼ - in-process gRPC backend :50051 (same container) -``` - -Because the inner daemon runs **inside** the trainer container: - -- **Networking is co-located.** Envoy's upstream `grpc-backend:host-gateway` - resolves to the trainer container itself — exactly where the gRPC backend - listens. Nothing about `:50051` needs to be published to the host. -- **Bind mounts just work.** The bundled compose mounts `envoy.yaml` and the - certs dir from the installed `weightslab` package. The inner daemon shares the - trainer container's filesystem, so those paths resolve with **no path - alignment** required (contrast this with the siblings variant). -- We only re-publish the **browser-facing** ports (`5173`, `8080`) from the - trainer container out to the host. - -The trade-off: the container needs `--privileged`, and you pay for a nested -daemon (an isolated image store, a second pull of Envoy + the frontend image). - -## Run it - -```bash -# from this directory -docker compose up --build -``` - -Then open . Stop with `Ctrl+C` (or `docker compose down`). - -### Use your local dev branch instead of PyPI - -By default the image installs `weightslab` from PyPI. To build against the dev -branch (e.g. to pick up uncommitted-to-PyPI CLI changes): - -```bash -docker compose build \ - --build-arg WEIGHTSLAB_SPEC="git+https://github.com/GrayboxTech/weightslab.git@dev" -docker compose up -``` - -(The repo must be reachable from the build — public, or pass build credentials.) - -### 🔒 Secured TLS (HTTPS + mTLS) - -DinD is the easiest option for TLS because the certs, Envoy, and the backend are -all co-located in one container. TLS is a **one-flag toggle** — just set -`WEIGHTSLAB_TLS=1`: - -```bash -WEIGHTSLAB_TLS=1 docker compose up --build -``` - -then **trust the dev CA on your host** (step 2 below) and open -**https://localhost:5173**. - -#### What the toggle does (in [entrypoint.sh](entrypoint.sh)) - -There are three pieces to a working TLS setup; the toggle handles the first two: - -1. **Generate certs + configure the UI** — runs `weightslab ui launch --certs`, - which generates the cert set into `WEIGHTSLAB_CERTS_DIR` - (`/root/.weightslab-certs`) and configures Envoy + the frontend for HTTPS. -2. **Turn TLS on for the backend** — `--certs` only configures Envoy + the - frontend, so the toggle also exports `GRPC_TLS_ENABLED=1` + - `GRPC_TLS_CERT_DIR=$WEIGHTSLAB_CERTS_DIR` before the example. Without this, - Envoy's upstream mTLS handshake fails against a plaintext backend (503s). - -The generator writes `ca.crt`, `envoy-server.*` (browser↔Envoy), `envoy-client.*` -+ `ca.crt` (Envoy↔backend mTLS), and `backend-server.*` (the backend's own cert, -CN `host.docker.internal`). `GRPC_TLS_CERT_DIR` makes the backend pick up -`backend-server.*` + `ca.crt` from there. - -> Optional: to also *enforce* the gRPC auth token the UI sends, export -> `GRPC_AUTH_TOKEN="$(cat $WEIGHTSLAB_CERTS_DIR/.grpc_auth_token)"` in the -> entrypoint (otherwise the token is simply ignored — TLS still encrypts). - -#### 2. Trust the dev CA on the Windows host - -The dev CA is generated *inside* the container, so your host browser doesn't -trust it yet. Export it and trust it (once): - -```powershell -# pull the CA out of the running container -docker cp weightslab_trainer_dind:/root/.weightslab-certs/ca.crt . -# trust it for your user, then restart the browser -Import-Certificate -FilePath .\ca.crt -CertStoreLocation Cert:\CurrentUser\Root -``` - -The certs carry SANs for `localhost`/`127.0.0.1`, so `https://localhost:5173` -(frontend) and `https://localhost:8080` (Envoy) validate once the CA is trusted. -Then open **https://localhost:5173**. (Skip this and TLS still works, but the -browser shows a self-signed warning.) - -> Without step 3 the connection is still encrypted, but the browser shows a -> self-signed warning. Without step 2, the UI loads over HTTPS but RPCs fail -> (Envoy can't complete mTLS to a plaintext backend). - -## Notes / gotchas - -- **`--privileged` is mandatory** for the inner `dockerd`. If your environment - forbids it, use the siblings variant instead. -- First run is slow: it pulls `envoyproxy/envoy` and `graybx/weightslab`, then - downloads MNIST for the example. The `dind_storage` volume caches images - across runs. -- The example trains until stopped (`training_steps_to_do: null`) but starts - paused (`is_training: false`); start/steer training from the UI. diff --git a/weightslab/examples/Docker_training/docker_in_docker/docker-compose.yml b/weightslab/examples/Docker_training/docker_in_docker/docker-compose.yml deleted file mode 100644 index 034c7a4d..00000000 --- a/weightslab/examples/Docker_training/docker_in_docker/docker-compose.yml +++ /dev/null @@ -1,54 +0,0 @@ -# ============================================================================= -# WeightsLab training docker — Docker-in-Docker (DinD) variant -# ============================================================================= -# One container that runs its own docker daemon, the WeightsLab CLI, and the -# training process. The Envoy + frontend containers live *inside* this -# container's daemon, so we re-publish their ports (8080, 5173) from this -# container out to the host for the browser. -# ============================================================================= -services: - trainer: - build: - context: . - # To use your local dev branch instead of PyPI, uncomment: - # args: - # WEIGHTSLAB_SPEC: "git+https://github.com/GrayboxTech/weightslab.git@dev" - image: weightslab-trainer-dind - container_name: weightslab_trainer_dind - # Required so the inner dockerd can run (DinD). - privileged: true - # --- GPU ------------------------------------------------------------------ - # Grant the host NVIDIA GPU(s) to the TRAINER container (the torch training - # runs here; the nested Envoy/frontend don't need a GPU). Requires the NVIDIA - # Container Toolkit on the host. NOTE: this is a hard requirement — on a host - # with no NVIDIA GPU/toolkit, comment out this whole `deploy:` block (the - # stack then runs on CPU; the entrypoint's nvidia-smi check degrades cleanly). - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all # or e.g. count: 1, or device_ids: ["0"] - capabilities: [gpu] - environment: - # Port the in-process gRPC backend binds to and Envoy dials. The inner - # Envoy reaches it via grpc-backend:host-gateway, which (inside DinD) - # resolves to THIS container — where the backend listens. No host - # publishing of 50051 is needed (contrast with the siblings variant). - - GRPC_BACKEND_PORT=50051 - # TLS toggle: run `WEIGHTSLAB_TLS=1 docker compose up` for HTTPS + mTLS - # (generates certs in-container; trust ca.crt on the host — see README). - - WEIGHTSLAB_TLS=${WEIGHTSLAB_TLS:-0} - ports: - # Re-publish the inner stack's browser-facing ports to the host. - - "5173:5173" # Weights Studio frontend -> http://localhost:5173 - - "8080:8080" # Envoy gRPC-web (browser -> backend) - volumes: - # Persist the inner daemon's image/layer store across restarts so we don't - # re-pull envoy + graybx/weightslab every run. - - dind_storage:/var/lib/docker - # The inner daemon + image pulls need a moment; give the stack room to start. - stop_grace_period: 30s - -volumes: - dind_storage: diff --git a/weightslab/examples/Docker_training/docker_in_docker/entrypoint.sh b/weightslab/examples/Docker_training/docker_in_docker/entrypoint.sh deleted file mode 100644 index ff1c85ae..00000000 --- a/weightslab/examples/Docker_training/docker_in_docker/entrypoint.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# ============================================================================= -# DinD entrypoint: start the inner docker daemon, launch the UI stack, then run -# the classification example in the foreground (it serves the gRPC backend). -# ============================================================================= -set -euo pipefail - -# Set WEIGHTSLAB_TLS=1 (via the trainer service environment) for HTTPS + mTLS. -WEIGHTSLAB_TLS="${WEIGHTSLAB_TLS:-0}" -# Single source of truth for certs — ui-launch generates here, the backend reads here. -export WEIGHTSLAB_CERTS_DIR="${WEIGHTSLAB_CERTS_DIR:-$HOME/.weightslab-certs}" - -# --- GPU visibility check (best-effort) -------------------------------------- -# Show the GPU the container can see. Non-fatal: on a CPU-only host (or one -# without the NVIDIA Container Toolkit) this just prints a notice and training -# falls back to CPU (config device: auto). -echo "[entrypoint] Checking GPU visibility (nvidia-smi)..." -if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then - nvidia-smi -L -else - echo "[entrypoint] No NVIDIA GPU visible (nvidia-smi unavailable) — running on CPU." -fi - -echo "[entrypoint] Starting inner docker daemon (DinD)..." -# The inner daemon manages the Envoy + frontend containers. It needs --privileged -# (set on the trainer service in docker-compose.yml). -dockerd >/var/log/dockerd.log 2>&1 & - -echo "[entrypoint] Waiting for inner docker daemon to become ready..." -for _ in $(seq 1 60); do - if docker info >/dev/null 2>&1; then - break - fi - sleep 1 -done -if ! docker info >/dev/null 2>&1; then - echo "[entrypoint] ERROR: inner dockerd did not start. Last log lines:" >&2 - tail -n 40 /var/log/dockerd.log >&2 || true - exit 1 -fi -echo "[entrypoint] Inner docker daemon is up." - -# The bundled deploy script would otherwise run a full 'docker compose build -# --no-cache' of the frontend. We only need the published image, so tell it to -# just write the .env and skip docker ops — 'ui launch' then pulls + starts the -# stack itself via 'docker compose up --pull always'. -export WEIGHTSLAB_SKIP_DOCKER_OPS=1 - -if [ "${WEIGHTSLAB_TLS}" = "1" ]; then - echo "[entrypoint] TLS mode — launching SECURED UI (HTTPS + mTLS) into ${WEIGHTSLAB_CERTS_DIR}..." - weightslab ui launch --certs - # --certs only configures Envoy + the frontend. The backend gRPC server needs - # TLS turned on too, or Envoy's upstream mTLS handshake fails (503s). Point it - # at the same cert dir (holds backend-server.crt/key + ca.crt). - export GRPC_TLS_ENABLED=1 - export GRPC_TLS_CERT_DIR="${WEIGHTSLAB_CERTS_DIR}" - SCHEME="https" -else - echo "[entrypoint] HTTP mode — launching UNSECURED UI..." - weightslab ui launch - SCHEME="http" -fi - -echo "[entrypoint] Starting the classification (cls) example — serves gRPC on :${GRPC_BACKEND_PORT:-50051}." -echo "[entrypoint] Open the UI at ${SCHEME}://localhost:5173 once training is serving." -# Foreground: keeps the container alive while the backend serves the browser. -exec weightslab start example # --3d_det diff --git a/weightslab/examples/Docker_training/siblings_self_contained/Dockerfile b/weightslab/examples/Docker_training/siblings_self_contained/Dockerfile deleted file mode 100644 index b66e2162..00000000 --- a/weightslab/examples/Docker_training/siblings_self_contained/Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -# ============================================================================= -# WeightsLab "training docker" — self-contained siblings (no host bind mounts) -# ============================================================================= -# Like the siblings variant, this drives the HOST docker daemon (mounted socket) -# so Envoy + frontend run as siblings. BUT it removes the host bind-mount -# dependency entirely: -# - Envoy's config is delivered through a named VOLUME, populated over the -# docker socket (no host path needed). -# - the frontend runs in HTTP mode, which needs NO certs mount at all. -# Consequence: NO path alignment, NO setup-host.sh, and it runs natively on -# Windows / Docker Desktop. The trade-off: it does NOT call `weightslab ui -# launch` — it brings the UI up from its own bind-mount-free compose -# (ui-compose.yml), reusing the same Envoy + graybx/weightslab images. -# ============================================================================= -FROM python:3.11-slim - -# --- System deps ------------------------------------------------------------- -# docker CLI + compose plugin ONLY (no daemon — we use the host's daemon). -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl sudo ca-certificates gnupg \ - && install -m 0755 -d /etc/apt/keyrings \ - && curl -fsSL https://download.docker.com/linux/debian/gpg \ - | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ - && chmod a+r /etc/apt/keyrings/docker.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ - https://download.docker.com/linux/debian $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ - > /etc/apt/sources.list.d/docker.list \ - && apt-get update && apt-get install -y --no-install-recommends \ - docker-ce-cli docker-compose-plugin \ - && rm -rf /var/lib/apt/lists/* - -# --- WeightsLab -------------------------------------------------------------- -# Plain pip install — NO editable/path-aligned install needed here, because the -# host daemon never has to resolve a package path (no host bind mounts). To run -# the dev branch instead: -# --build-arg WEIGHTSLAB_SPEC="git+https://github.com/GrayboxTech/weightslab.git@dev" -ARG WEIGHTSLAB_SPEC=weightslab -RUN pip install --no-cache-dir "${WEIGHTSLAB_SPEC}" - -ENV GRPC_BACKEND_PORT=50051 - -# --- GPU (NVIDIA) ------------------------------------------------------------ -# Make the NVIDIA Container Toolkit inject the host driver (nvidia-smi + libs). -# No-op without an NVIDIA GPU/toolkit; the actual grant is in docker-compose.yml. -ENV NVIDIA_VISIBLE_DEVICES=all \ - NVIDIA_DRIVER_CAPABILITIES=compute,utility - -# The bind-mount-free UI compose, read by the docker CLI inside this container -# and sent to the host daemon. It references only images + a named volume. -COPY ui-compose.yml /opt/ui/ui-compose.yml -COPY entrypoint.sh /usr/local/bin/entrypoint.sh -RUN sed -i 's/\r$//' /usr/local/bin/entrypoint.sh \ - && chmod +x /usr/local/bin/entrypoint.sh - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/weightslab/examples/Docker_training/siblings_self_contained/README.md b/weightslab/examples/Docker_training/siblings_self_contained/README.md deleted file mode 100644 index 6cd927cd..00000000 --- a/weightslab/examples/Docker_training/siblings_self_contained/README.md +++ /dev/null @@ -1,159 +0,0 @@ -# Training docker — Option C: self-contained siblings (no host bind mounts) - -Like Option B (siblings / Docker-out-of-Docker) the trainer drives the **host** -docker daemon (mounted socket) so Envoy + frontend run as **siblings**. But this -variant **removes every host bind-mount dependency**, so it runs **entirely from -inside the container** — no `setup-host.sh`, no path alignment, and it works -**natively on Windows / Docker Desktop**. - -Inside the trainer container, in order: - -1. render Envoy's plaintext config from the installed `weightslab` package, -2. push it into a **named volume over the docker socket** (no host path), -3. bring up Envoy + frontend from a **bind-mount-free** [ui-compose.yml](ui-compose.yml), -4. `weightslab start example --cls` → serve the **gRPC backend** on `:50051`. - -End result: open `http://localhost:5173` and the UI talks to training end-to-end. - -## The trade-off vs Option B - -Option B uses the stock **`weightslab ui launch`**, whose bundled compose -bind-mounts `envoy.yaml` + certs from the package — which the **host** daemon -must resolve, forcing path alignment + `setup-host.sh` (and breaking on Windows). - -Option C instead **does not call `weightslab ui launch`**. It brings the UI up -itself from [ui-compose.yml](ui-compose.yml), which has **no host bind mounts**: - -| What the bundled compose bind-mounts | How Option C delivers it without a host path | -|---|---| -| `envoy.yaml` (+ plaintext variants) | rendered from the installed package, then piped into the **`wl_envoy_cfg` named volume** over the socket (`docker run -i … busybox`) | -| `${WEIGHTSLAB_CERTS_DIR}` → Envoy/nginx | **omitted** — the frontend's nginx auto-falls back to **HTTP** when no certs are present; Envoy uses the fully-plaintext template | - -It still reuses the **same images** (`envoyproxy/envoy`, `graybx/weightslab`) and -the **same Envoy template** that ships with weightslab, so it stays in sync. - -> By default this path is **HTTP-only**. It *can* be secured with TLS using the -> same named-volume trick to deliver certs (no host bind mounts) — see -> [🔒 Enabling TLS](#-enabling-tls-https--mtls) below. - -## How the wiring works - -``` -host browser - → localhost:5173 ──► frontend container (sibling, HTTP, no certs) - → localhost:8080 ──► Envoy container (sibling, config from named volume) - │ grpc-backend:host-gateway → host:50051 - ▼ - host:50051 ──► trainer container's gRPC backend :50051 -``` - -Same networking as Option B — the trainer **publishes `:50051`** so the sibling -Envoy reaches the in-container backend via `host-gateway`. Only the Envoy + -frontend publish `8080`/`5173` (from `ui-compose.yml`). - -## ⚙️ Required configuration - -| # | Requirement | Where | Why | -|---|---|---|---| -| 1 | **Mount `/var/run/docker.sock`** | [docker-compose.yml](docker-compose.yml) | drive the host daemon (siblings) | -| 2 | **Publish `50051:50051`** from the trainer | [docker-compose.yml](docker-compose.yml) | sibling Envoy dials it via `host-gateway` | -| 3 | **Stage Envoy config into a named volume over the socket** | [entrypoint.sh](entrypoint.sh) | replaces the host bind mount — no host path | -| 4 | **`ui-compose.yml` with no host bind mounts** (named volume + HTTP frontend) | [ui-compose.yml](ui-compose.yml) | nothing for the host daemon to resolve on disk | -| 5 | **`extra_hosts: grpc-backend:host-gateway`** on Envoy | [ui-compose.yml](ui-compose.yml) | route Envoy → host:50051 → trainer | -| 6 | **`GRPC_BACKEND_PORT=50051`** (matches the rendered Envoy config) | [docker-compose.yml](docker-compose.yml) / [Dockerfile](Dockerfile) | port the backend binds + Envoy dials | -| — | *Not needed (unlike Option B):* path alignment, `setup-host.sh`, a host source checkout | — | the daemon never resolves a host path | - -### GPU — to run `nvidia-smi` / train on CUDA - -| Requirement | Where | -|---|---| -| Host: NVIDIA driver **+ NVIDIA Container Toolkit** (`sudo nvidia-ctk runtime configure --runtime=docker`) | host setup | -| `deploy.resources.reservations.devices` (driver `nvidia`, `capabilities: [gpu]`) | [docker-compose.yml](docker-compose.yml) | -| `NVIDIA_VISIBLE_DEVICES=all` + `NVIDIA_DRIVER_CAPABILITIES=compute,utility` | [Dockerfile](Dockerfile) | - -> The `deploy:` GPU block is a **hard requirement**: on a host with no NVIDIA -> GPU/toolkit, comment it out in [docker-compose.yml](docker-compose.yml) or -> `up` fails. Only the **trainer** needs the GPU. - -## Run it - -```bash -# from this directory — works on Windows/Docker Desktop, no host prep -docker compose up --build -``` - -Then open . - -### Stopping - -```bash -docker compose down # stops the trainer -# the sibling UI stack is a separate project: -docker compose -p weightslab_ui -f ui-compose.yml down -docker volume rm wl_envoy_cfg # optional: drop the staged config -``` - -## 🔒 Enabling TLS (HTTPS + mTLS) - -TLS works here too, **still with no host bind mounts** — the certs are delivered -the same way as `envoy.yaml`: piped into named volumes over the docker socket. -It's a **one-flag toggle**: - -```bash -WEIGHTSLAB_TLS=1 docker compose up --build -``` - -then **trust the dev CA on your host** (step 2 below) and open -**https://localhost:5173**. - -#### What the toggle does (in [entrypoint.sh](entrypoint.sh)) - -In TLS mode the entrypoint generates one cert set with `weightslab se` (writes -`ca.crt`, `envoy-server.*`, `envoy-client.*`, `backend-server.*` + -`.grpc_auth_token` into `WEIGHTSLAB_CERTS_DIR`) and wires up four layers — all -fed over the socket, no host paths: - -| Layer | Needs | Delivered via | -|---|---|---| -| Browser ↔ Envoy (downstream) | `envoy-server.crt/key` | `wl_envoy_cfg` volume → `/etc/envoy/certs` | -| Envoy ↔ backend (upstream mTLS) | `envoy-client.crt/key` + `ca.crt` | `wl_envoy_cfg` volume → `/etc/envoy/certs` | -| Browser ↔ frontend (nginx HTTPS) | `envoy-server.crt/key` | `wl_nginx_certs` volume → `/etc/nginx/certs` | -| Backend gRPC server | `backend-server.crt/key` + `ca.crt` | `GRPC_TLS_ENABLED=1` + `GRPC_TLS_CERT_DIR` | - -Concretely, vs the HTTP path, the toggle: - -1. runs `weightslab se` and renders the **full mTLS** template - (`ui/envoy/envoy.yaml`, not the `*_plaintext` one) into `wl_envoy_cfg`; -2. stages the Envoy certs into `wl_envoy_cfg:/etc/envoy/certs` and the frontend - cert into `wl_nginx_certs` (and `chmod a+rX` them — the generated keys are - `0600`/root, and Envoy/nginx run non-root, so without this Envoy crashes with - *"unable to read file …envoy-client.key"*); -3. sets `WS_SERVER_PROTOCOL=https` so the frontend's nginx serves HTTPS - ([ui-compose.yml](ui-compose.yml) always mounts `wl_nginx_certs`, empty in - HTTP mode); -4. exports `GRPC_TLS_ENABLED=1` + `GRPC_TLS_CERT_DIR=$WEIGHTSLAB_CERTS_DIR` so - the backend serves gRPC over TLS (else Envoy's upstream mTLS fails → 503s). - -> Optional: to *enforce* the gRPC auth token, also export -> `GRPC_AUTH_TOKEN="$(cat $WEIGHTSLAB_CERTS_DIR/.grpc_auth_token)"` (otherwise the -> token the UI sends is ignored — TLS still encrypts). - -#### 2. Trust the dev CA on the Windows host - -The dev CA is generated *inside* the container, so trust it on the host once: - -```powershell -docker cp weightslab_trainer_selfcontained:/root/.weightslab-certs/ca.crt . -Import-Certificate -FilePath .\ca.crt -CertStoreLocation Cert:\CurrentUser\Root -``` - -Then open **https://localhost:5173**. Certs carry `localhost`/`127.0.0.1` SANs, so -the frontend (5173) and Envoy (8080) both validate once the CA is trusted. -(Skip this and TLS still works, but the browser shows a self-signed warning.) - -## Notes - -- First run pulls `envoyproxy/envoy` + `graybx/weightslab` into the host image - cache and downloads MNIST for the example. -- HTTP by default; TLS is opt-in (section above). -- The example starts paused (`is_training: false`); start/steer it from the UI. diff --git a/weightslab/examples/Docker_training/siblings_self_contained/docker-compose.yml b/weightslab/examples/Docker_training/siblings_self_contained/docker-compose.yml deleted file mode 100644 index 3ef00358..00000000 --- a/weightslab/examples/Docker_training/siblings_self_contained/docker-compose.yml +++ /dev/null @@ -1,44 +0,0 @@ -# ============================================================================= -# WeightsLab training docker — self-contained siblings (no host bind mounts) -# ============================================================================= -# The trainer drives the HOST docker daemon (mounted socket) to start Envoy + -# frontend as siblings — but with ZERO host bind mounts and NO path alignment, -# so it runs natively on Windows / Docker Desktop and needs no setup-host.sh. -# The gRPC backend runs in THIS container and is published to the host so the -# sibling Envoy can reach it via host-gateway. -# ============================================================================= -services: - trainer: - build: - context: . - # Dev branch instead of PyPI: - # args: - # WEIGHTSLAB_SPEC: "git+https://github.com/GrayboxTech/weightslab.git@dev" - image: weightslab-trainer-selfcontained - container_name: weightslab_trainer_selfcontained - # --- GPU ------------------------------------------------------------------ - # Grant the host NVIDIA GPU(s) to the trainer (torch runs here; the sibling - # Envoy/frontend don't need a GPU). Requires the NVIDIA Container Toolkit. - # NOTE: hard requirement — on a host with no NVIDIA GPU/toolkit, comment out - # this `deploy:` block (the stack then runs on CPU). - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all # or e.g. count: 1, or device_ids: ["0"] - capabilities: [gpu] - environment: - - GRPC_BACKEND_PORT=50051 - # TLS toggle: run `WEIGHTSLAB_TLS=1 docker compose up` for HTTPS + mTLS - # (certs generated in-container + delivered via named volumes; trust - # ca.crt on the host — see README "Enabling TLS"). - - WEIGHTSLAB_TLS=${WEIGHTSLAB_TLS:-0} - ports: - # Publish the gRPC backend so the sibling Envoy reaches it via - # grpc-backend:host-gateway -> host:50051 -> this container. Envoy + the - # frontend publish 8080/5173 to the host themselves (see ui-compose.yml). - - "50051:50051" - volumes: - # Only mount: the host docker socket. NO source/cert bind mounts. - - /var/run/docker.sock:/var/run/docker.sock diff --git a/weightslab/examples/Docker_training/siblings_self_contained/entrypoint.sh b/weightslab/examples/Docker_training/siblings_self_contained/entrypoint.sh deleted file mode 100644 index 5259f9b0..00000000 --- a/weightslab/examples/Docker_training/siblings_self_contained/entrypoint.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# ============================================================================= -# Self-contained siblings entrypoint — runs entirely from inside the container, -# with NO host bind mounts and NO path alignment. -# -# Flow: -# 1. render Envoy's plaintext config from the installed weightslab package, -# 2. push it into a named volume *over the docker socket* (no host path), -# 3. bring up Envoy + frontend (ui-compose.yml) on the host daemon, -# 4. run the example (serves the gRPC backend, published to the host). -# ============================================================================= -set -euo pipefail - -GRPC_BACKEND_PORT="${GRPC_BACKEND_PORT:-50051}" -ENVOY_VOLUME="wl_envoy_cfg" -NGINX_CERTS_VOLUME="wl_nginx_certs" -UI_COMPOSE="/opt/ui/ui-compose.yml" -UI_PROJECT="weightslab_ui" - -# Set WEIGHTSLAB_TLS=1 (via the trainer service environment) for HTTPS + mTLS. -WEIGHTSLAB_TLS="${WEIGHTSLAB_TLS:-0}" -export WEIGHTSLAB_CERTS_DIR="${WEIGHTSLAB_CERTS_DIR:-$HOME/.weightslab-certs}" - -# --- GPU visibility check (best-effort) -------------------------------------- -echo "[entrypoint] Checking GPU visibility (nvidia-smi)..." -if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then - nvidia-smi -L -else - echo "[entrypoint] No NVIDIA GPU visible (nvidia-smi unavailable) — running on CPU." -fi - -# --- Reach the host docker daemon -------------------------------------------- -if ! docker info >/dev/null 2>&1; then - echo "[entrypoint] ERROR: cannot reach the host docker daemon." >&2 - echo " Did you mount /var/run/docker.sock? (see docker-compose.yml)" >&2 - exit 1 -fi - -# Locate the installed package's Envoy templates. find_spec does NOT execute -# weightslab/__init__.py, so it avoids the import-time logging banner that would -# otherwise pollute this command substitution. -PKG="$(python -c 'import importlib.util as u; print(u.find_spec("weightslab").submodule_search_locations[0])')" - -# --- 0) Start from clean named volumes --------------------------------------- -# Tear down any prior UI stack first so the volumes are free to recreate. -docker compose -p "${UI_PROJECT}" -f "${UI_COMPOSE}" down --remove-orphans >/dev/null 2>&1 || true -docker rm -f weights_studio_envoy weights_studio_frontend >/dev/null 2>&1 || true -docker volume rm "${ENVOY_VOLUME}" "${NGINX_CERTS_VOLUME}" >/dev/null 2>&1 || true -docker volume create "${ENVOY_VOLUME}" >/dev/null -docker volume create "${NGINX_CERTS_VOLUME}" >/dev/null - -# Helper: pipe a file from this container into a named volume over the socket -# (no host path involved). Usage: stage_file -stage_file() { docker run --rm -i -v "$1:/vol" busybox sh -c "cat > /vol/$2" < "$3"; } -# Helper: copy named files from WEIGHTSLAB_CERTS_DIR into / via tar. -# The generated key files are 0600/root; Envoy + nginx run as non-root, so make -# them readable (dev certs) — otherwise Envoy crashes with "unable to read file". -stage_certs() { local vol="$1" sub="$2"; shift 2 - tar -C "${WEIGHTSLAB_CERTS_DIR}" -cf - "$@" \ - | docker run --rm -i -v "${vol}:/vol" busybox \ - sh -c "mkdir -p /vol/${sub} && tar -C /vol/${sub} -xf - && chmod -R a+rX /vol/${sub}"; } - -if [ "${WEIGHTSLAB_TLS}" = "1" ]; then - echo "[entrypoint] TLS mode — generating certs and staging the mTLS Envoy config..." - # Generate ca.crt, envoy-server.*, envoy-client.*, backend-server.* + token. - weightslab se - ENVOY_TMPL="${PKG}/ui/envoy/envoy.yaml" # FULL mTLS template - [ -f "${ENVOY_TMPL}" ] || { echo "[entrypoint] ERROR: ${ENVOY_TMPL} not found" >&2; exit 1; } - sed "s/__GRPC_BACKEND_PORT__/${GRPC_BACKEND_PORT}/g" "${ENVOY_TMPL}" > /tmp/envoy.generated.yaml - stage_file "${ENVOY_VOLUME}" "envoy.yaml" /tmp/envoy.generated.yaml - # Envoy's certs (downstream server + upstream client + CA) -> /etc/envoy/certs - stage_certs "${ENVOY_VOLUME}" "certs" ca.crt envoy-server.crt envoy-server.key envoy-client.crt envoy-client.key - # The frontend (nginx) HTTPS cert -> /etc/nginx/certs (triggers HTTPS mode) - stage_certs "${NGINX_CERTS_VOLUME}" "." envoy-server.crt envoy-server.key - export WS_SERVER_PROTOCOL=https - # Backend gRPC TLS (else Envoy's upstream mTLS fails against a plaintext backend). - export GRPC_TLS_ENABLED=1 - export GRPC_TLS_CERT_DIR="${WEIGHTSLAB_CERTS_DIR}" - SCHEME="https" -else - echo "[entrypoint] HTTP mode — staging the plaintext Envoy config..." - ENVOY_TMPL="${PKG}/ui/envoy/envoy.downstream_upstream_plaintext.yaml" - [ -f "${ENVOY_TMPL}" ] || { echo "[entrypoint] ERROR: ${ENVOY_TMPL} not found" >&2; exit 1; } - sed "s/__GRPC_BACKEND_PORT__/${GRPC_BACKEND_PORT}/g" "${ENVOY_TMPL}" > /tmp/envoy.generated.yaml - stage_file "${ENVOY_VOLUME}" "envoy.yaml" /tmp/envoy.generated.yaml - # nginx certs volume left empty -> frontend serves HTTP. - export WS_SERVER_PROTOCOL=http - SCHEME="http" -fi -echo "[entrypoint] Staged Envoy config (backend port ${GRPC_BACKEND_PORT}, scheme ${SCHEME})." - -# --- Bring up Envoy + frontend as siblings on the host daemon ---------------- -echo "[entrypoint] Starting Envoy + frontend (bind-mount-free) on the host daemon..." -WS_SERVER_PROTOCOL="${WS_SERVER_PROTOCOL}" \ - docker compose -p "${UI_PROJECT}" -f "${UI_COMPOSE}" up -d --pull always - -echo "[entrypoint] UI is up — open ${SCHEME}://localhost:5173 once training is serving." - -# --- Run the example — serves gRPC on :GRPC_BACKEND_PORT (published) ---------- -echo "[entrypoint] Starting the classification (cls) example — serves gRPC on :${GRPC_BACKEND_PORT}." -exec weightslab start example --cls diff --git a/weightslab/examples/Docker_training/siblings_self_contained/ui-compose.yml b/weightslab/examples/Docker_training/siblings_self_contained/ui-compose.yml deleted file mode 100644 index 52f1f06c..00000000 --- a/weightslab/examples/Docker_training/siblings_self_contained/ui-compose.yml +++ /dev/null @@ -1,61 +0,0 @@ -# ============================================================================= -# Bind-mount-free UI stack (Envoy + Weights Studio frontend), run on the HOST -# daemon by the trainer container's entrypoint. The whole point: NO host bind -# mounts, so the host daemon needs nothing from any host path. -# - Envoy reads its config from the `wl_envoy_cfg` named volume, which the -# entrypoint populates over the docker socket (see entrypoint.sh). -# - the frontend runs in HTTP mode and needs no certs mount. -# This reuses the exact same images as `weightslab ui launch`. -# ============================================================================= -services: - envoy: - image: envoyproxy/envoy:v1.28-latest - container_name: weights_studio_envoy - # Config comes from the named volume (already port-substituted by the - # entrypoint), so just point Envoy at it — no in-container templating. - command: ["/usr/local/bin/envoy", "-c", "/etc/envoy/envoy.yaml"] - volumes: - - wl_envoy_cfg:/etc/envoy:ro - extra_hosts: - # Envoy is a sibling on the host; it dials the trainer's gRPC backend via - # the host (host-gateway → host:50051, which the trainer publishes). - - "grpc-backend:host-gateway" - - "host.docker.internal:host-gateway" - ports: - - "8080:8080" # gRPC-web (browser -> backend) - networks: - - weights_studio_network - restart: unless-stopped - - frontend: - image: graybx/weightslab:latest - container_name: weights_studio_frontend - # The image's nginx entrypoint serves HTTPS when certs are present in - # /etc/nginx/certs, else HTTP. The trainer entrypoint populates the - # wl_nginx_certs volume only in TLS mode (empty otherwise -> HTTP), and sets - # WS_SERVER_PROTOCOL accordingly. - environment: - - WS_SERVER_PROTOCOL=${WS_SERVER_PROTOCOL:-http} - - WL_ENABLE_GRPC_AUTH_TOKEN=0 - - GRPC_AUTH_TOKEN= - volumes: - - wl_nginx_certs:/etc/nginx/certs:ro - ports: - - "5173:443" # frontend -> http(s)://localhost:5173 - depends_on: - - envoy - networks: - - weights_studio_network - restart: unless-stopped - -networks: - weights_studio_network: - driver: bridge - -volumes: - # Created + populated by the trainer entrypoint over the docker socket. - wl_envoy_cfg: - external: true - # Frontend HTTPS certs — populated only in TLS mode (empty -> HTTP). - wl_nginx_certs: - external: true diff --git a/weightslab/examples/Lightning/ws-classification/config.yaml b/weightslab/examples/Lightning/wl-classification/config.yaml similarity index 100% rename from weightslab/examples/Lightning/ws-classification/config.yaml rename to weightslab/examples/Lightning/wl-classification/config.yaml diff --git a/weightslab/examples/Lightning/ws-classification/main.py b/weightslab/examples/Lightning/wl-classification/main.py similarity index 100% rename from weightslab/examples/Lightning/ws-classification/main.py rename to weightslab/examples/Lightning/wl-classification/main.py diff --git a/weightslab/examples/Notebooks/Colab/wl-colab-quickstart.ipynb b/weightslab/examples/Notebooks/Colab/wl-colab-quickstart.ipynb new file mode 100644 index 00000000..2fe94105 --- /dev/null +++ b/weightslab/examples/Notebooks/Colab/wl-colab-quickstart.ipynb @@ -0,0 +1,77 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Welcome to the WeightsLab Colab quickstart — a blank starter for streaming your own training run into Weights Studio.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# WeightsLab · Colab quickstart\n", + "\n", + "> **This notebook is read-only.** Run it as-is to try it, or use **File ▸ Save a copy in Drive** to make it your own and edit freely.\n", + "\n", + "Colab gives you a free GPU; **Weights Studio runs on your own machine** and connects to this Colab backend through a public [`bore`](https://github.com/ekzhang/bore) tunnel.\n", + "\n", + "### What the cell below does\n", + "1. Installs WeightsLab.\n", + "2. Imports it as `wl`.\n", + "3. (Your turn) registers your model / optimizer / dataloaders with `wl.watch_or_edit(...)`.\n", + "4. Serves the backend over a bore tunnel and prints the `weightslab tunnel ...` command to run locally." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 1 · Install WeightsLab (+ Colab compatibility overrides)\n", + "%pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ --extra-index-url https://download.pytorch.org/whl/cpu weightslab==1.4.0.dev3\n", + "\n", + "# 2 · Import\n", + "import weightslab as wl\n", + "\n", + "# 3 · Serve to Weights Studio over a public bore tunnel.\n", + "# Prints a `bore.pub:PORT` endpoint. Then, on your own machine (Docker running):\n", + "# weightslab start # opens http://localhost:5173\n", + "# weightslab tunnel bore.pub:PORT # in another window - PORT = the port printed below\n", + "wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "# 4 · Register YOUR training objects here (uncomment and fill in):\n", + "# wl.watch_or_edit(model, optimizer, train_loader, test_loader)\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "wl-colab-quickstart.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/PyTorch/wl-ads-recommendation.ipynb b/weightslab/examples/Notebooks/PyTorch/wl-ads-recommendation.ipynb new file mode 100644 index 00000000..d09b94f9 --- /dev/null +++ b/weightslab/examples/Notebooks/PyTorch/wl-ads-recommendation.ipynb @@ -0,0 +1,481 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Welcome to the WeightsLab image-classification notebook! WeightsLab is an open-source PyTorch tool for dataset debugging, mislabel detection, and mid-training data curation. Browse the Docs for details, and raise an issue on GitHub for support.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advertising CTR Recommendation with WeightsLab (tabular)\n", + "\n", + "This notebook trains a **Wide & Deep** click-through-rate (CTR) model — the core of an advertising recommender — and instruments it with WeightsLab so every signal traces **back to the exact impression**.\n", + "\n", + "A sample is one ad impression (a **row**); the model input **is** the packed field vector (8 embedded categorical fields + 8 numeric features), which WeightsLab sends to the UI as a `vector`. Each field (`ad_category`, `placement`, `bid_price`, …) is a **sortable column** in the List view.\n", + "\n", + "### What you'll do\n", + "1. Install WeightsLab.\n", + "2. Generate reproducible ad impressions at a calibrated ~20% CTR.\n", + "3. Wrap the Wide & Deep model, optimizer, dataloaders, loss and metric.\n", + "4. Train while streaming per-impression signals to Weights Studio.\n", + "\n", + "*Real drop-in datasets: Criteo, Avazu, MovieLens.*" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Install WeightsLab from PyPI. These tabular demos are tiny — the free Colab CPU runtime is plenty (no GPU needed).\n", + "\n", + "> The **tabular input path** (the feature vector is sent to the UI as a `vector` — not a fake image) needs a WeightsLab build with tabular support (the `252-tabular-experiment` line or a release that includes it)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"PyPI\n", + "\"PyPI\n", + "\"PyPI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install torch torchvision" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev6\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Imports\n", + "\n", + "`weightslab` is imported as `wl`. The two `guard_*_context` managers scope a block as training vs. evaluation so signals are attributed to the right phase." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile, logging\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "from torch.utils.data import Dataset\n", + "from torchmetrics.classification import Accuracy\n", + "from tqdm.auto import tqdm\n", + "\n", + "import weightslab as wl\n", + "from weightslab.components.global_monitoring import (\n", + " guard_training_context,\n", + " guard_testing_context,\n", + ")\n", + "\n", + "logging.basicConfig(level=logging.ERROR)\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "print(f\"Using device: {device}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. A synthetic CTR dataset (rows, not images)\n", + "\n", + "`make_synthetic_ctr` builds impressions with 8 categorical fields + 8 numeric features and a binary `clicked` label at ~20% CTR, driven by per-field effects plus an `ad_category x user_segment` interaction. The dataset returns the **standardized** field vector (categorical indices + per-feature standardized numerics) as the model input, and `get_items(...)` exposes the **raw**, human-readable field values (categorical labels, dollars, years, page counts, …) as metadata columns. The **Wide & Deep** model embeds each categorical field and MLPs over the embeddings + numeric features." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "CATEGORICAL_FIELDS = [\"user_segment\", \"ad_category\", \"device_type\", \"os\",\n", + " \"publisher\", \"placement\", \"region\", \"hour_bucket\"]\n", + "CARD = [6, 10, 4, 4, 12, 5, 8, 6]\n", + "NUMERIC_FIELDS = [\"ad_position\", \"bid_price\", \"user_age\", \"session_depth\",\n", + " \"historical_ctr\", \"days_since_last_click\", \"num_ads_seen_today\",\n", + " \"creative_freshness\"]\n", + "NUM_CAT, NUM_NUM = len(CATEGORICAL_FIELDS), len(NUMERIC_FIELDS)\n", + "VOCABS = {\"device_type\": [\"mobile\", \"desktop\", \"tablet\", \"ctv\"],\n", + " \"os\": [\"android\", \"ios\", \"windows\", \"other\"],\n", + " \"placement\": [\"banner\", \"sidebar\", \"interstitial\", \"native\", \"video\"],\n", + " \"hour_bucket\": [\"night\", \"early\", \"morning\", \"midday\", \"afternoon\", \"evening\"]}\n", + "TARGET_CTR = 0.20\n", + "\n", + "\n", + "def _label(field, code):\n", + " v = VOCABS.get(field)\n", + " return v[code] if v and 0 <= code < len(v) else f\"{field}_{code}\"\n", + "\n", + "\n", + "def _draw_raw_numeric(rng, n):\n", + " \"\"\"8 numeric fields at their natural, human-readable scale (dollars, years, counts, ...).\"\"\"\n", + " return np.stack([\n", + " rng.integers(1, 9, n).astype(np.float64), # ad_position: slot 1-8\n", + " rng.gamma(2.0, 1.25, n), # bid_price: $, mean ~2.50\n", + " np.clip(rng.normal(35.0, 12.0, n), 18, 80), # user_age\n", + " rng.poisson(3.0, n).astype(np.float64) + 1, # session_depth (pages)\n", + " rng.beta(2.0, 25.0, n), # historical_ctr, mean ~7%\n", + " rng.exponential(10.0, n), # days_since_last_click\n", + " rng.poisson(12.0, n).astype(np.float64) + 1, # num_ads_seen_today\n", + " rng.exponential(20.0, n), # creative_freshness (days)\n", + " ], axis=1)\n", + "\n", + "\n", + "def make_synthetic_ctr(n, seed=0):\n", + " \"\"\"Return (cat[int], num_std[float], num_raw[float], y): shared ground-truth CTR model (seed 12345).\"\"\"\n", + " p = np.random.default_rng(12345)\n", + " cat_w = [p.normal(0, 0.8, c) for c in CARD]\n", + " num_w = p.normal(0, 0.5, NUM_NUM)\n", + " inter = p.normal(0, 0.7, (CARD[1], CARD[0]))\n", + " rng = np.random.default_rng(seed)\n", + " cat = np.stack([rng.integers(0, c, n) for c in CARD], axis=1).astype(np.int64)\n", + " num_raw = _draw_raw_numeric(rng, n)\n", + " mean, std = num_raw.mean(0, keepdims=True), num_raw.std(0, keepdims=True)\n", + " std[std == 0] = 1.0\n", + " num_std = (num_raw - mean) / std # standardized -> model input\n", + " logit = sum(w[cat[:, f]] for f, w in enumerate(cat_w)) + num_std @ num_w\n", + " logit = logit + inter[cat[:, 1], cat[:, 0]]\n", + " logit -= logit.mean()\n", + " lo, hi = -20.0, 20.0 # calibrate bias to hit ~20% mean CTR\n", + " for _ in range(60):\n", + " mid = 0.5 * (lo + hi)\n", + " if (1 / (1 + np.exp(-(logit + mid)))).mean() < TARGET_CTR:\n", + " lo = mid\n", + " else:\n", + " hi = mid\n", + " logit += 0.5 * (lo + hi)\n", + " y = (rng.uniform(0, 1, n) < 1 / (1 + np.exp(-logit))).astype(np.int64)\n", + " return cat, num_std.astype(np.float32), num_raw.astype(np.float32), y\n", + "\n", + "\n", + "class AdsCTRDataset(Dataset):\n", + " \"\"\"Yields (packed_vector, id, label); get_items() adds raw (non-standardized) field columns.\"\"\"\n", + "\n", + " def __init__(self, n, seed=0):\n", + " cat, num_std, num_raw, y = make_synthetic_ctr(n, seed)\n", + " self.cat, self.num, self.num_raw = cat, num_std, num_raw # self.num: standardized (model input)\n", + " self.features = torch.from_numpy(np.concatenate([cat.astype(np.float32), num_std], 1))\n", + " self.labels = torch.from_numpy(y)\n", + "\n", + " def __len__(self):\n", + " return len(self.features)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.features[idx], idx, int(self.labels[idx])\n", + "\n", + " def get_items(self, idx, include_metadata=False, include_labels=False, include_images=False):\n", + " x = self.features[idx] if include_images else None\n", + " target = int(self.labels[idx]) if include_labels else None\n", + " meta = None\n", + " if include_metadata:\n", + " meta = {n: _label(n, int(self.cat[idx][f])) for f, n in enumerate(CATEGORICAL_FIELDS)}\n", + " meta.update({n: round(float(self.num_raw[idx][j]), 4) for j, n in enumerate(NUMERIC_FIELDS)})\n", + " return x, idx, target, meta\n", + "\n", + "\n", + "class WideDeepCTR(nn.Module):\n", + " def __init__(self, card=CARD, num_numeric=NUM_NUM, emb=8, hidden=128, num_classes=2):\n", + " super().__init__()\n", + " self.card = list(card)\n", + " self.emb = nn.ModuleList([nn.Embedding(c, emb) for c in self.card])\n", + " self.deep = nn.Sequential(\n", + " nn.Linear(emb * len(self.card) + num_numeric, hidden), nn.ReLU(), nn.Dropout(0.1),\n", + " nn.Linear(hidden, hidden // 2), nn.ReLU(), nn.Linear(hidden // 2, num_classes))\n", + " self.wide_cat = nn.ModuleList([nn.Embedding(c, num_classes) for c in self.card])\n", + " self.wide_num = nn.Linear(num_numeric, num_classes)\n", + "\n", + " def forward(self, x):\n", + " flat = x.reshape(x.shape[0], -1)\n", + " cidx = flat[:, :len(self.card)].round().long()\n", + " cidx = cidx.clamp(min=torch.zeros(len(self.card), device=x.device, dtype=torch.long),\n", + " max=torch.tensor(self.card, device=x.device) - 1)\n", + " numeric = flat[:, len(self.card):]\n", + " deep = self.deep(torch.cat([e(cidx[:, i]) for i, e in enumerate(self.emb)] + [numeric], 1))\n", + " wide = self.wide_num(numeric)\n", + " for i, e in enumerate(self.wide_cat):\n", + " wide = wide + e(cidx[:, i])\n", + " return deep + wide\n", + "\n", + "\n", + "def build_dataset(n, seed=0):\n", + " return AdsCTRDataset(n, seed)\n", + "\n", + "\n", + "def build_model():\n", + " return WideDeepCTR()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Configuration\n", + "\n", + "Every tunable lives here in one dict, like a `config.yaml` with comments. Wrapping it with `flag=\"hyperparameters\"` lets Weights Studio read (and live-edit) these values while training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = {\n", + " \"experiment_name\": \"ads_ctr_wide_deep\",\n", + " \"device\": str(device),\n", + " \"root_log_dir\": tempfile.mkdtemp(prefix=\"weightslab_ads_\"),\n", + " \"learning_rate\": 0.005,\n", + " \"training_steps_to_do\": 3000,\n", + " \"eval_full_to_train_steps_ratio\": 150,\n", + " \"write_export_ratio\": 500,\n", + " \"class_weights\": [1.0, 4.0], # [no-click, click] — up-weight clicks\n", + " \"dataset\": {\"seed\": 0, \"n_train\": 8000, \"n_test\": 2000},\n", + " \"data\": {\"train_loader\": {\"batch_size\": 64},\n", + " \"test_loader\": {\"batch_size\": 256}},\n", + "}\n", + "wl.watch_or_edit(config, flag=\"hyperparameters\", poll_interval=1.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Wrap the training objects\n", + "\n", + "This is the heart of WeightsLab. Each object passes through `wl.watch_or_edit(...)` with a `flag` describing its role. The tracked dataset's `get_items()` exposes every feature as a **sortable column**; `preload_metadata=True` loads them at init." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cfg = config\n", + "train_ds = build_dataset(cfg['dataset']['n_train'], seed=cfg['dataset']['seed'])\n", + "test_ds = build_dataset(cfg['dataset']['n_test'], seed=cfg['dataset']['seed'] + 1)\n", + "\n", + "model = wl.watch_or_edit(build_model().to(device), flag='model', device=device)\n", + "optimizer = wl.watch_or_edit(\n", + " optim.Adam(model.parameters(), lr=cfg['learning_rate']), flag='optimizer')\n", + "\n", + "train_loader = wl.watch_or_edit(\n", + " train_ds, flag='data', loader_name='train_loader',\n", + " batch_size=cfg['data']['train_loader']['batch_size'], shuffle=True,\n", + " is_training=True, preload_labels=True, preload_metadata=True)\n", + "test_loader = wl.watch_or_edit(\n", + " test_ds, flag='data', loader_name='test_loader',\n", + " batch_size=cfg['data']['test_loader']['batch_size'], shuffle=False,\n", + " is_training=False, preload_labels=True, preload_metadata=True)\n", + "\n", + "# Class weights counter the minority (positive) class prevalence.\n", + "cw = torch.tensor(cfg['class_weights'], dtype=torch.float32, device=device)\n", + "train_criterion = wl.watch_or_edit(\n", + " nn.CrossEntropyLoss(weight=cw, reduction='none'),\n", + " flag='loss', signal_name='train-loss-CE', log=True)\n", + "test_criterion = wl.watch_or_edit(\n", + " nn.CrossEntropyLoss(weight=cw, reduction='none'),\n", + " flag='loss', signal_name='test-loss-CE', log=True)\n", + "metric = wl.watch_or_edit(\n", + " Accuracy(task='multiclass', num_classes=2).to(device),\n", + " flag='metric', signal_name='metric-ACC', log=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Train and evaluate steps\n", + "\n", + "The `guard_training_context` / `guard_testing_context` blocks tell WeightsLab the phase. `criterion(..., batch_ids=ids, preds=preds)` passes the sample ids so loss is stored **per sample**, and `wl.save_signals(...)` logs a custom per-sample accuracy signal during evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def train(loader, model, optimizer, criterion, device):\n", + " with guard_training_context:\n", + " inputs, ids, labels = next(loader)\n", + " inputs, labels = inputs.to(device), labels.to(device)\n", + " optimizer.zero_grad()\n", + " logits = model(inputs)\n", + " preds = logits.argmax(dim=1, keepdim=True)\n", + " loss = criterion(logits.float(), labels.long(), batch_ids=ids, preds=preds)\n", + " total = loss.mean()\n", + " total.backward()\n", + " optimizer.step()\n", + " return total.detach().cpu().item()\n", + "\n", + "\n", + "def test(loader, model, criterion, metric, device, n_batches):\n", + " losses = torch.tensor(0.0, device=device)\n", + " for inputs, ids, labels in loader:\n", + " with guard_testing_context:\n", + " inputs, labels = inputs.to(device), labels.to(device)\n", + " logits = model(inputs)\n", + " preds = logits.argmax(dim=1, keepdim=True)\n", + " losses += criterion(logits, labels, batch_ids=ids, preds=preds).mean()\n", + " metric.update(logits, labels)\n", + " correct = (preds.view(-1) == labels.view(-1)).float()\n", + " wl.save_signals(preds_raw=logits, targets=labels, batch_ids=ids,\n", + " signals={\"test_metric/Accuracy_per_sample\": correct}, preds=preds)\n", + " return (losses / max(1, n_batches)).item(), (metric.compute() * 100).item()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Serve and train\n", + "\n", + "`wl.serve(serving_grpc=True, serving_bore=True)` starts the background gRPC server (non-blocking) and a `bore.pub` tunnel so Weights Studio on your own machine can reach this Colab backend. `wl.start_training(...)` flips the experiment into the *training* state, then we run the loop." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wl.serve(serving_grpc=True, serving_bore=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wl.start_training(timeout=3)\n", + "\n", + "steps = config['training_steps_to_do']\n", + "eval_ratio = config['eval_full_to_train_steps_ratio']\n", + "n_test_batches = len(test_loader)\n", + "\n", + "test_loss = test_acc = None\n", + "pbar = tqdm(range(steps), desc='Training')\n", + "for step in pbar:\n", + " age = model.get_age() if hasattr(model, 'get_age') else step\n", + " train_loss = train(train_loader, model, optimizer, train_criterion, device)\n", + " if age > 0 and age % eval_ratio == 0:\n", + " test_loss, test_acc = test(test_loader, model, test_criterion, metric, device, n_test_batches)\n", + " postfix = {'loss': f'{train_loss:.3f}'}\n", + " if test_acc is not None:\n", + " postfix['test_acc'] = f'{test_acc:.1f}%'\n", + " pbar.set_postfix(postfix)\n", + "\n", + "wl.write_history()\n", + "wl.write_dataframe()\n", + "print('Training complete.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## See it live in Weights Studio\n", + "\n", + "Everything above runs headless. The payoff is **Weights Studio**, where each row is one record and every feature is a **sortable column**.\n", + "\n", + "Studio runs as a local Docker stack, and **Colab has no Docker daemon**, so you run Studio on your own machine and point it at this notebook's backend via the `bore.pub:` endpoint **printed in Section 6**.\n", + "\n", + "**On your machine** (with Docker Desktop):\n", + "```bash\n", + "pip install weightslab\n", + "weightslab start # opens http://localhost:5173\n", + "weightslab tunnel bore.pub:12345 # in another window, the host:port printed in Section 6\n", + "```\n", + "\n", + "Then open **http://localhost:5173** and switch the Data Exploration board to the **List** view." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Curate in the UI\n", + "\n", + "In Weights Studio, switch the Data Exploration board to the **List** view:\n", + "\n", + "1. **Sort by `test-loss-CE` descending** and generate its histogram to find the impressions the model ranks worst.\n", + "2. **Lock** the loss sort, then add `ad_category` or `placement` as a secondary sort to see which segments the model struggles on.\n", + "3. **Right-click a column** to clone, reset the sort, or generate a histogram.\n", + "4. **Discard** noisy impressions and keep training to sharpen the ranker." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "
\n", + "Crafted by GrayboxTech - if WeightsLab helps you catch a bad label, drop us a star on GitHub.\n", + "
" + ] + } + ], + "metadata": { + "colab": { + "include_colab_link": true, + "name": "wl-ads-recommendation.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/PyTorch/wl-classification.ipynb b/weightslab/examples/Notebooks/PyTorch/wl-classification.ipynb new file mode 100644 index 00000000..d6b9859a --- /dev/null +++ b/weightslab/examples/Notebooks/PyTorch/wl-classification.ipynb @@ -0,0 +1,1667 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "view-in-github" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d1Yi--O7FETe" + }, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Welcome to the WeightsLab image-classification notebook! WeightsLab is an open-source PyTorch tool for dataset debugging, mislabel detection, and mid-training data curation. Browse the Docs for details, and raise an issue on GitHub for support.
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "POIbm5qMFETh" + }, + "source": [ + "# Image Classification with WeightsLab\n", + "\n", + "This notebook trains a small CNN on **MNIST** and instruments it with WeightsLab so every training signal is traced **back to the exact samples** producing it.\n", + "\n", + "Most data problems (mislabels, outliers, class imbalance) stay invisible until your model tells you through the loss. By wrapping your training objects with `wl.watch_or_edit(...)`, WeightsLab records **per-sample** loss and metrics live, so you can rank the worst samples, spot bad labels, and curate the dataset **without restarting training**.\n", + "\n", + "### What you'll do\n", + "1. Install WeightsLab.\n", + "2. Set every knob in one **config** dict (like a `config.yaml`).\n", + "3. Wrap the model, optimizer, dataloaders, loss, and metric with the SDK.\n", + "4. Train while per-sample signals are captured.\n", + "5. Rank the highest-loss samples inline, and (optionally) open the live **Weights Studio** UI." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P6gYbKq5FETh" + }, + "source": [ + "## Setup\n", + "\n", + "Install WeightsLab from PyPI. On Colab the free **T4 GPU** runtime is plenty for this demo (`Runtime -> Change runtime type -> T4 GPU`)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gRQuuS90FETi" + }, + "source": [ + "\"PyPI\n", + "\"PyPI\n", + "\"PyPI" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HW0Mrb23FETi", + "outputId": "d649cd01-8f95-4599-a5c1-d10ff377964e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: torch in /usr/local/lib/python3.12/dist-packages (2.9.0)\n", + "Requirement already satisfied: torchvision in /usr/local/lib/python3.12/dist-packages (0.24.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch) (3.29.4)\n", + "Requirement already satisfied: typing-extensions>=4.10.0 in /usr/local/lib/python3.12/dist-packages (from torch) (4.15.0)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch) (3.5.0)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.12/dist-packages (from torchvision) (2.0.2)\n", + "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.12/dist-packages (from torchvision) (11.3.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch) (1.3.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch) (3.0.3)\n" + ] + } + ], + "source": [ + "%pip install torch torchvision" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nQscI28FGE4Z" + }, + "source": [] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "PVreJLYJIod6", + "outputId": "9e77d75f-ef33-4807-a6c6-c69bb8e4a29a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in indexes: https://test.pypi.org/simple/, https://pypi.org/simple/\n", + "\u001b[31mERROR: Could not find a version that satisfies the requirement weightslab==1.3.3.dev6 (from versions: 0.0.0, 1.1.1, 1.1.2, 1.1.3.dev0, 1.1.3, 1.1.4, 1.1.5, 1.1.6, 1.1.7.dev3, 1.1.7, 1.1.8, 1.1.9, 1.2.0, 1.2.1.dev0, 1.2.1, 1.2.2, 1.2.3.dev0, 1.2.3, 1.2.4, 1.2.5, 1.2.6, 1.3.0.dev0, 1.3.0, 1.3.1.dev1, 1.3.1.dev2, 1.3.1.dev3, 1.3.1, 1.3.2.dev0, 1.3.2, 1.3.3.dev0, 1.3.3.dev1, 1.3.3.dev2, 1.3.3.dev3, 1.3.3.dev4, 1.3.3.dev5, 1.3.3, 1.9.1.dev1, 1.9.1.dev2, 1.9.1.dev3, 1.9.1.dev4, 20260427.dev1, 20260427.dev2, 20260605.dev1, 20260310110810, 20260310112657.dev2659810514, 20260310112936.dev2035376017, 20260310114507.dev36735734, 20260310122733.dev571756140, 20260310130048.dev500189762, 20260310142756.dev2389960633, 20260310172643.dev1790296826, 20260310184554.dev3572723963, 20260310190409.dev16171358, 20260311001628.dev82842326, 20260311001938.dev237981957, 20260311094631.dev2433054235, 20260311100556.dev2759443978, 20260311111421.dev1419930837, 20260311152850.dev375337088, 20260415152016.dev1529547259, 20260515094038.dev295368024, 20260515094810.dev295368024, 20260515100544.dev709462904, 20260518115533.dev1831822349, 20260518120723.dev3543420334, 20260518121158.dev2454375021, 20260518125331.dev2144318793, 20260518152946.dev2688573905, 20260518153709.dev457831340, 20260518154038.dev3380177004, 20260518154550.dev2597851767, 20260518160221.dev67736641, 20260518160351.dev500880749, 20260518160424.dev3971069761, 20260519121616.dev3380177004, 20260519122038.dev2371413592, 20260519153002.dev2989691760, 20260519160247.dev4080968838, 20260519160445.dev1656037640, 20260519160647.dev1656037640, 20260519161900.dev3430116169, 20260520150146.dev2702511536, 20260520151257.dev3739150100, 20260520151549.dev3739150100, 20260520160858.dev3843696446, 20260520161330.dev950128852, 20260521100142.dev3061163053, 20260521100237.dev3763458477, 20260521100304.dev2708183098, 20260521101422.dev1111427347, 20260521101942.dev1751890316, 20260521102210.dev697219115, 20260521112427.dev1815380353, 20260528101213.dev1156931848, 20260528101621.dev3002779928, 20260528102838.dev1225638335, 20260528104815.dev146316976, 20260528104912.dev2432105952, 20260528144750.dev228483024, 20260529124654.dev1088603485, 20260602102247.dev3028334291, 20260602133031.dev612843591, 20260603164444.dev1140145833, 20260604132920.dev4091502713, 20260604141626.dev180590797, 20260604154956.dev1777185457, 20260604161731.dev1492593462, 20260605101712.dev3201556731, 20260605102734.dev307817433, 20260605153744.dev1218078270, 20260605154211.dev1284003932, 20260605155425.dev2249789890, 20260608122954.dev616519929, 20260608130348.dev103073330, 20260608142704.dev1750894834, 20260608143725.dev279338996, 20260608144113.dev1776325482, 20260608151812.dev1493920450, 20260609145543.dev4094763614, 20260609150002.dev1824924853, 20260609150942.dev1916930876, 20260609151231.dev2007644555, 20260609164013.dev3320873074, 20260609170327.dev427086775, 20260609170903.dev4090827397, 20260609170910.dev878249345, 20260609171010.dev315170550, 20260610134434.dev2867115601, 20260610135809.dev2659845748, 20260610140822.dev3917850211, 20260610142006.dev1779011735, 20260610145358.dev827089089, 20260611100101.dev2435580864, 20260611122329.dev2942846670, 20260612094419.dev318051075, 20260612174112.dev1945936079, 20260612175512.dev558737864, 20260612175811.dev4293577391, 20260614170924.dev435351911, 20260617105222.dev3481250246, 20260617105337.dev4027879073, 20260617150202.dev2472499979, 20260617150338.dev3507587562, 20260618091144.dev3638457713, 20260618091310.dev2980634153, 20260618091437.dev666220125, 20260618161401.dev2672707490, 20260618161455.dev1062308657, 20260618161503.dev869421065, 20260619154839.dev3124849231, 20260619154914.dev1825884740, 20260619155132.dev3059317318, 20260619155148.dev2195359802, 20260619155831.dev2547505056, 20260619160048.dev2036037367, 20260619160133.dev887552290, 20260619164552.dev545225221, 20260620125803.dev57300911, 20260620155947.dev163709547, 20260620160335.dev978317758, 20260620161239.dev1614881684, 20260620163624.dev3186857944, 20260620163931.dev174614870, 20260620170427.dev937264444, 20260621140029.dev2748921241, 20260622131949.dev1993441264, 20260622132042.dev41563737, 20260626162054.dev815684860, 20260626162149.dev1517909447, 20260629095034.dev2864733358, 20260629095109.dev2365834914, 20260629100925.dev3295139208, 20260705162125.dev3118493333, 20260705162448.dev4054744437, 20260705162608.dev112194667, 20260705163739.dev337968460, 20260705163915.dev1702540858, 20260706102840.dev1428624302, 20260706103324.dev4063235919, 20260706103956.dev1761667072, 20260706105255.dev612849548, 20260706122808.dev397480003, 20260706123234.dev2929755776, 20260706124917.dev3042106597, 20260706132510.dev3579337194, 20260707110009.dev944740769, 20260707110041.dev949537649, 20260707130443.dev1726432854, 20260707130503.dev2999317701, 20260707130900.dev2393574598, 20260708143958.dev3881013472, 20260708144357.dev578891966, 20260708144457.dev658501659, 20260708144826.dev1757704307, 20260708144934.dev3945788796, 20260708145114.dev663145800, 20260710162317.dev3300605375, 20260710162608.dev3232863539)\u001b[0m\u001b[31m\n", + "\u001b[0m\u001b[31mERROR: No matching distribution found for weightslab==1.3.3.dev6\u001b[0m\u001b[31m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "%pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev6\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pKzQf_BzFETj" + }, + "source": [ + "## 1. Imports\n", + "\n", + "`weightslab` is imported as `wl`. The two `guard_*_context` managers scope a block as training vs. evaluation so signals are attributed to the right phase." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-uAGOoqVFETj", + "outputId": "1d806744-f7be-4da8-9a8c-4b68ae436a13" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "15/07/2026-14:06:00.794 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmpn4he5yzg/weightslab_logs/weightslab_20260715_140600.log\n", + "15/07/2026-14:06:00.796 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "15/07/2026-14:06:00.797 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev5 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "Using device: cuda\n" + ] + } + ], + "source": [ + "import os\n", + "import tempfile\n", + "import logging\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "from torch.utils.data import Dataset\n", + "from torchvision import datasets, transforms\n", + "from torchmetrics.classification import Accuracy\n", + "from tqdm.auto import tqdm\n", + "\n", + "import weightslab as wl\n", + "from weightslab.examples.utils.baseline_models.pytorch.models import FashionCNN as CNN\n", + "from weightslab.components.global_monitoring import (\n", + " guard_training_context,\n", + " guard_testing_context,\n", + ")\n", + "\n", + "logging.basicConfig(level=logging.ERROR)\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "print(f\"Using device: {device}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bwyu_n8VFETj" + }, + "source": [ + "## 2. A dataset that carries sample identity\n", + "\n", + "WeightsLab attributes every signal to a **sample id**. This thin wrapper around torchvision's MNIST returns `(image, id, label)` and attaches a virtual filepath per sample, so signals can be traced back to individual images in the UI." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "NoYmyRYwFETk" + }, + "outputs": [], + "source": [ + "class MNISTCustomDataset(Dataset):\n", + " \"\"\"MNIST that returns (image, index, label) and tracks a virtual filepath.\"\"\"\n", + "\n", + " def __init__(self, root, train=True, download=False, transform=None):\n", + " self.mnist = datasets.MNIST(root=root, train=train, download=download, transform=None)\n", + " self.transform = transform\n", + " self.train = train\n", + " split = \"train\" if train else \"test\"\n", + " self.filepaths = {}\n", + " for idx in range(len(self.mnist)):\n", + " label = self.mnist.targets[idx].item()\n", + " self.filepaths[idx] = os.path.join(\n", + " \"MNIST\", \"processed\", split, f\"class_{label}\", f\"sample_{idx:05d}.pt\"\n", + " )\n", + "\n", + " def __len__(self):\n", + " return len(self.mnist)\n", + "\n", + " def __getitem__(self, idx):\n", + " image, label = self.mnist[idx]\n", + " if self.transform:\n", + " image = self.transform(image)\n", + " return image, idx, label" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J-jUVL-cFETk" + }, + "source": [ + "## 3. Configuration\n", + "\n", + "Every tunable lives here, in one dict, like a `config.yaml` with comments. Wrapping it with `flag=\"hyperparameters\"` lets the Studio UI read (and live-edit) these values while training." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pHaJYoqNFETk", + "outputId": "6bae5503-1a48-48c3-de01-8534009157a8" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "15/07/2026-14:23:06.251 INFO:weightslab.components.checkpoint_manager:__init__: CheckpointManager initialized at /tmp/weightslab_mnist_yonuysxx\n", + "15/07/2026-14:23:06.253 INFO:weightslab.src:watch_or_edit: Registered new checkpoint manager in ledger\n", + "Experiment logs -> /tmp/weightslab_mnist_yonuysxx\n" + ] + } + ], + "source": [ + "log_dir = tempfile.mkdtemp(prefix=\"weightslab_mnist_\")\n", + "\n", + "config = {\n", + " # -- Experiment -------------------------------------------------------\n", + " \"experiment_name\": \"mnist_classification\", # name shown in Weights Studio\n", + " \"device\": str(device), # \"cuda\" if a GPU is available, else \"cpu\"\n", + " \"root_log_dir\": log_dir, # where signal history / dataframes are written\n", + " \"num_classes\": 10, # MNIST digits 0-9\n", + "\n", + " # -- Training schedule ------------------------------------------------\n", + " \"training_steps_to_do\": 8000, # total optimizer steps (raise for a longer live run)\n", + " \"eval_full_to_train_steps_ratio\": 500, # run a full eval every N steps\n", + " \"experiment_dump_to_train_steps_ratio\": 250, # dump model weights ratio\n", + " \"write_export_ratio\": 1000, # export signal history + dataframe every N steps\n", + "\n", + " # Configure global dataframe storage\n", + " \"ledger_enable_flushing_threads\": True,\n", + " \"ledger_enable_h5_persistence\": True,\n", + " \"ledger_flush_max_rows\": 1024,\n", + " \"ledger_flush_interval\": 20.0,\n", + "\n", + " # -- Optimizer --------------------------------------------------------\n", + " \"learning_rate\": 0.001, # Adam learning rate\n", + "\n", + " # -- Data loaders -----------------------------------------------------\n", + " # One block per loader, keyed by loader_name. EVERY kwarg passed to\n", + " # wl.watch_or_edit(..., flag=\"data\") lives here so the wrap cell below stays\n", + " # declarative — no dataloader settings hardcoded in the cells. Add a new\n", + " # loader by adding another block here.\n", + " \"data\": {\n", + " \"train_loader\": {\n", + " \"batch_size\": 128, # training batch size\n", + " \"shuffle\": True, # shuffle each epoch\n", + " \"is_training\": True, # marks this loader as the training split\n", + " \"compute_hash\": False, # skip per-sample content hashing (faster init)\n", + " \"preload_labels\": True, # preload labels into the ledger at startup\n", + " \"preload_metadata\": False, # load metadata lazily on first access\n", + " \"enable_h5_persistence\": True, # persist per-sample stats to the H5 store\n", + " },\n", + " \"test_loader\": {\n", + " \"batch_size\": 128, # evaluation batch size\n", + " \"shuffle\": False, # keep test order stable\n", + " \"is_training\": False, # marks this loader as an eval split\n", + " \"compute_hash\": False,\n", + " \"preload_labels\": True,\n", + " \"preload_metadata\": False,\n", + " \"enable_h5_persistence\": True,\n", + " },\n", + " },\n", + "\n", + " # -- Services ---------------------------------------------------------\n", + " \"serving_grpc\": True, # expose the gRPC backend for Weights Studio\n", + " \"serving_bore\": True, # expose the bore tunnel for Weights Studio\n", + "}\n", + "\n", + "wl.watch_or_edit(config, flag=\"hyperparameters\", poll_interval=1.0)\n", + "print(\"Experiment logs ->\", log_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5jiuWkNBFETk" + }, + "source": [ + "## 4. Wrap the training objects\n", + "\n", + "This is the heart of WeightsLab. Each object is passed through `wl.watch_or_edit(...)` with a `flag` describing its role. The returned objects behave exactly like the originals, but now report their state and per-sample signals to WeightsLab." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "d7HMuDrCFETk", + "outputId": "b5bbf0b2-9464-46ce-dc04-4841e15d0c97" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "15/07/2026-14:06:03.264 INFO:weightslab.backend.model_interface:__init__: Using checkpoint manager from ledger\n", + "15/07/2026-14:06:03.266 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loading checkpoint 124b3608e98f2d94...\n", + "15/07/2026-14:06:03.267 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Target: HP=124b3608 MODEL=e98f2d94 DATA=9e70c5a5\n", + "15/07/2026-14:06:03.268 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Current: HP=124b3608 MODEL=e98f2d94 DATA=9e70c5a5\n", + "15/07/2026-14:06:03.269 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Model architecture unchanged, using current model\n", + "15/07/2026-14:06:03.292 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [OK] Loaded weights from step 6750 with RNG state\n", + "15/07/2026-14:06:03.294 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Config unchanged, using current config\n", + "15/07/2026-14:06:03.296 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loaded components: {'weights'}\n", + "15/07/2026-14:06:03.300 INFO:weightslab.backend.model_interface:__init__: Auto-loaded model weights from checkpoint 124b3608e98f2d94 (step 6750)\n", + "15/07/2026-14:06:03.304 INFO:weightslab.data.data_samples_with_ops:__init__: [DataSampleTrackingWrapper] H5 persistence enabled at /tmp/weightslab_mnist_kg8215pb/checkpoints/data/data.h5\n", + "15/07/2026-14:06:03.402 INFO:weightslab.data.data_samples_with_ops:__init__: Preloading sample statistics for PHYSICAL indices in split 'train_loader' with: preload_labels=True,\n", + "15/07/2026-14:06:03.404 INFO:weightslab.data.data_samples_with_ops:__init__: Metadata will be loaded on demand from the wrapped dataset for split 'train_loader' when accessed, which may increase latency on first access but reduces initialization time.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Initializing ledger for split 'train_loader': 100%|██████████| 60000/60000 [00:12<00:00, 4884.89it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "15/07/2026-14:06:15.708 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] Registering split 'train_loader' with 60000 samples.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "15/07/2026-14:06:16.062 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] After annotation expansion: 60000 samples → 60000 annotation rows.\n", + "15/07/2026-14:06:16.064 INFO:weightslab.data.h5_array_store:__init__: [H5ArrayStore] Initialized with cache limit: 2048MB\n", + "15/07/2026-14:06:17.387 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loading checkpoint 124b3608e98f2d94...\n", + "15/07/2026-14:06:17.389 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Target: HP=124b3608 MODEL=e98f2d94 DATA=9e70c5a5\n", + "15/07/2026-14:06:17.390 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Current: HP=124b3608 MODEL=e98f2d94 DATA=9e70c5a5\n", + "15/07/2026-14:06:17.393 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Model architecture unchanged, using current model\n", + "15/07/2026-14:06:17.393 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Config unchanged, using current config\n", + "15/07/2026-14:06:17.593 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [OK] Loaded data snapshot (70000 rows) with RNG state\n", + "15/07/2026-14:06:17.594 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loaded components: {'data'}\n", + "15/07/2026-14:06:17.918 INFO:weightslab.backend.dataloader_interface:_load_checkpoint_data: Applied data snapshot from checkpoint (70000 rows)\n", + "15/07/2026-14:06:17.922 INFO:weightslab.data.data_samples_with_ops:__init__: [DataSampleTrackingWrapper] H5 persistence enabled at /tmp/weightslab_mnist_kg8215pb/checkpoints/data/data.h5\n", + "15/07/2026-14:06:17.932 INFO:weightslab.data.data_samples_with_ops:__init__: Preloading sample statistics for PHYSICAL indices in split 'test_loader' with: preload_labels=True,\n", + "15/07/2026-14:06:17.935 INFO:weightslab.data.data_samples_with_ops:__init__: Metadata will be loaded on demand from the wrapped dataset for split 'test_loader' when accessed, which may increase latency on first access but reduces initialization time.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Initializing ledger for split 'test_loader': 100%|██████████| 10000/10000 [00:01<00:00, 7865.73it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "15/07/2026-14:06:19.213 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] Registering split 'test_loader' with 10000 samples.\n", + "15/07/2026-14:06:19.273 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] After annotation expansion: 10000 samples → 10000 annotation rows.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "15/07/2026-14:06:19.798 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loading checkpoint 124b3608e98f2d94...\n", + "15/07/2026-14:06:19.799 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Target: HP=124b3608 MODEL=e98f2d94 DATA=9e70c5a5\n", + "15/07/2026-14:06:19.802 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Current: HP=124b3608 MODEL=e98f2d94 DATA=9e70c5a5\n", + "15/07/2026-14:06:19.803 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Model architecture unchanged, using current model\n", + "15/07/2026-14:06:19.804 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Config unchanged, using current config\n", + "15/07/2026-14:06:19.965 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [OK] Loaded data snapshot (70000 rows) with RNG state\n", + "15/07/2026-14:06:19.966 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loaded components: {'data'}\n", + "15/07/2026-14:06:20.277 INFO:weightslab.backend.dataloader_interface:_load_checkpoint_data: Applied data snapshot from checkpoint (70000 rows)\n" + ] + } + ], + "source": [ + "data_root = os.path.join(log_dir, \"data\")\n", + "os.makedirs(data_root, exist_ok=True)\n", + "\n", + "train_ds = MNISTCustomDataset(root=data_root, train=True, download=True,\n", + " transform=transforms.ToTensor())\n", + "test_ds = MNISTCustomDataset(root=data_root, train=False, download=True,\n", + " transform=transforms.ToTensor())\n", + "\n", + "# Model + optimizer\n", + "model = wl.watch_or_edit(CNN().to(device), flag=\"model\", device=device, compute_dependencies=True)\n", + "optimizer = wl.watch_or_edit(\n", + " optim.Adam(model.parameters(), lr=config[\"learning_rate\"]), flag=\"optimizer\")\n", + "\n", + "# Tracked dataloaders — all loader settings come from config[\"data\"][],\n", + "# so nothing is hardcoded here. Unpack each block straight into the wrapper.\n", + "train_loader = wl.watch_or_edit(\n", + " train_ds, flag=\"data\", loader_name=\"train_loader\",\n", + " **config[\"data\"][\"train_loader\"],\n", + ")\n", + "test_loader = wl.watch_or_edit(\n", + " test_ds, flag=\"data\", loader_name=\"test_loader\",\n", + " **config[\"data\"][\"test_loader\"],\n", + ")\n", + "\n", + "# Watched losses + metric (they log themselves per sample)\n", + "train_criterion = wl.watch_or_edit(nn.CrossEntropyLoss(reduction=\"none\"),\n", + " flag=\"loss\", signal_name=\"train-loss-CE\", log=True)\n", + "test_criterion = wl.watch_or_edit(nn.CrossEntropyLoss(reduction=\"none\"),\n", + " flag=\"loss\", signal_name=\"test-loss-CE\", log=True)\n", + "metric = wl.watch_or_edit(\n", + " Accuracy(task=\"multiclass\", num_classes=config[\"num_classes\"]).to(device),\n", + " flag=\"metric\", signal_name=\"metric-ACC\", log=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-nWmYOC1FETk" + }, + "source": [ + "## 5. Train and evaluate steps\n", + "\n", + "The `guard_training_context` / `guard_testing_context` blocks tell WeightsLab which phase it's in. `criterion(..., batch_ids=ids, preds=preds)` passes the sample ids so the loss is stored **per sample**, and `wl.save_signals(...)` logs a custom per-sample accuracy signal during evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "LFFmdEMTFETl" + }, + "outputs": [], + "source": [ + "def train(loader, model, optimizer, criterion, device):\n", + " with guard_training_context:\n", + " inputs, ids, labels = next(loader)\n", + " inputs, labels = inputs.to(device), labels.to(device)\n", + "\n", + " optimizer.zero_grad()\n", + " logits = model(inputs)\n", + " preds = logits.argmax(dim=1, keepdim=True)\n", + "\n", + " loss = criterion(logits.float(), labels.long(), batch_ids=ids, preds=preds)\n", + " total_loss = loss.mean()\n", + " total_loss.backward()\n", + " optimizer.step()\n", + " return total_loss.detach().cpu().item()\n", + "\n", + "\n", + "import time\n", + "def test(loader, model, criterion, metric, device, n_batches):\n", + " losses = torch.tensor(0.0, device=device)\n", + " st = time.time()\n", + " for inputs, ids, labels in loader:\n", + " with guard_testing_context:\n", + " inputs, labels = inputs.to(device), labels.to(device)\n", + " logits = model(inputs)\n", + " preds = logits.argmax(dim=1, keepdim=True)\n", + "\n", + " loss = criterion(logits, labels, batch_ids=ids, preds=preds)\n", + " losses += loss.mean()\n", + " metric.update(logits, labels)\n", + "\n", + " correct = (preds.view(-1) == labels.view(-1)).float()\n", + " wl.save_signals(\n", + " preds_raw=logits, targets=labels, batch_ids=ids, preds=preds,\n", + " signals={\n", + " \"test_metric/Accuracy_per_sample\": correct,\n", + " \"test_metric/Inverse_Accuracy_per_sample\": 1.0 - correct,\n", + " },\n", + " )\n", + " print(f\"Compute test in {time.time()-st}s\")\n", + " return (losses / n_batches).item(), (metric.compute() * 100).item()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nBNH0B0-FETl" + }, + "source": [ + "## 6. Serve and train\n", + "\n", + "`wl.serve(serving_grpc=True)` starts the background gRPC server (non-blocking) that Weights Studio connects to. `wl.start_training(...)` flips the experiment into the *training* state, then we run the loop, periodically evaluating and exporting signals." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 886 + }, + "id": "YnEB98kvRBJt", + "outputId": "590e5572-23b7-4bc1-8592-b666ce7f71dc" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "15/07/2026-14:06:20.303 INFO:weightslab.trainer.trainer_services:_run_security_preflight: \n", + "\n", + "# #######################################\n", + "# #######################################\n", + "[gRPC] Security preflight checks:\n", + "\tTLS: DISABLED (unencrypted traffic)\n", + "\tAuth tokens: NONE configured\n", + "\t! WARNING: GRPC_TLS_ENABLED=0. Traffic will be unencrypted. Use only for development.\n", + "\t! WARNING: No GRPC_AUTH_TOKEN/GRPC_AUTH_TOKENS configured. Only transport-level trust (TLS/mTLS) will protect RPC access.\n", + "# #######################################\n", + "# #######################################\n", + "\n", + "15/07/2026-14:06:20.305 INFO:weightslab.trainer.trainer_services:grpc_serve: [gRPC] Watchdogs disabled via WEIGHTSLAB_DISABLE_WATCHDOGS.\n", + "15/07/2026-14:06:20.306 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Thread callback started\n", + "15/07/2026-14:06:20.307 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Creating ThreadPoolExecutor with 6 worker threads (n_workers_grpc=None, max_concurrent_rpcs=None)\n", + "15/07/2026-14:06:20.308 INFO:weightslab.trainer.trainer_services:grpc_serve: [gRPC] Server started with watchdogs disabled (host=0.0.0.0 port=50051 workers=None)\n", + "15/07/2026-14:06:20.315 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Server object created\n", + "15/07/2026-14:06:20.525 INFO:weightslab.trainer.services.agent.agent:__init__: Initializing DataManipulationAgent\n", + "15/07/2026-14:06:20.623 WARNING:weightslab.trainer.services.agent.agent:_load_config: Error loading config from /usr/local/lib/python3.12/dist-packages: [Errno 21] Is a directory: '/usr/local/lib/python3.12/dist-packages'\n", + "15/07/2026-14:06:20.625 INFO:weightslab.trainer.services.agent.agent:_load_config: \n", + "\n", + "# #######################################\n", + "# #######################################\n", + "Agent initialized from configuration /content/agent_config.yaml: \n", + "\tFinal Agent Configuration: Preferred Provider=openrouter, \n", + "\tFallback to Local=True, \n", + "\tOpenRouter Model=~google/gemini-flash-latest with:\n", + "\t\tAPI Key=None\n", + "\t\tBase URL=https://openrouter.ai/api/v1, \n", + "\tOllama Model=llama3.2:3b\n", + "# #######################################\n", + "# #######################################\n", + "\n", + "15/07/2026-14:06:20.625 INFO:weightslab.trainer.services.agent.agent:_setup_providers: Setting up Ollama with model llama3.2:3b\n", + "15/07/2026-14:06:20.710 INFO:weightslab.trainer.services.agent.agent:_setup_providers: [Agent] Ollama enabled: llama3.2:3b\n", + "15/07/2026-14:06:20.711 INFO:weightslab.trainer.services.data_service:_build_preview_cache: [PreviewCache] Building 64×64 or less or less preview cache for 2000 samples …\n", + "15/07/2026-14:06:20.712 INFO:weightslab.trainer.services.data_service:__init__: DataService initialized.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r[PreviewCache]: 0%| | 0/2000 [00:00` endpoint **printed in Section 6**.\n", + "\n", + "**On your machine** (with Docker Desktop):\n", + "```bash\n", + "pip install weightslab\n", + "\n", + "# Terminal 1 - launch the UI (plaintext HTTP, the default)\n", + "weightslab ui launch # opens http://localhost:5173\n", + "\n", + "# Terminal 2 - bridge the Colab backend to localhost:50051\n", + "weightslab tunnel bore.pub:12345 # in another window, the host:port printed in Section 6\n", + "```\n", + "\n", + "Then open **http://localhost:5173** - Studio connects through your local Envoy -> `weightslab tunnel` -> `bore.pub` -> this Colab backend, and you watch training stream live.\n", + "\n", + "> Note: keep it plaintext end-to-end (the default `weightslab ui launch`, raw-TCP `bore`) so gRPC's HTTP/2 frames pass through untouched.\n", + "\n", + "Prefer to keep it all local? Run this same example on your own machine (`weightslab start example --cls`) and launch the UI next to it - no tunnel required." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 10. Curate in the UI to boost performance\n", + "\n", + "This is where WeightsLab pays off: instead of training longer on all 60k samples, use the live signals to **find the samples that matter, keep only those, and fine-tune**. Everything below happens in Weights Studio while the backend keeps running — no code changes, no restart.\n", + "\n", + "> **Baseline before curating:** the evaluation metric (test accuracy) sits around **0.86**.\n", + "\n", + "**1. See more of the dataset.** In the **grid explorer**, increase the number of images per page so you can scan many samples at once.\n", + "\n", + "**2. Find the easy and hard samples.** Sort the grid by **`train-loss-CE` in decreasing order**, then **generate the histogram** for that signal. The distribution is bimodal — a tail of **hard** examples (training loss **> 2.45**) and a bulk of **easy** ones (training loss **< 1.461**). These two extremes carry the signal; the vast middle is largely redundant.\n", + "\n", + "**3. Let the agent tag them.** Initialize the **agent** (the panel in the UI), then ask it:\n", + "\n", + "> *Tag training samples with train loss greater than 2.45 as \"hard_ex\" and training samples with train loss lower than 1.45 as \"easy_ex\".*\n", + "\n", + "Inspect the new `tag:hard_ex` / `tag:easy_ex` columns in the grid to confirm the tagging looks right.\n", + "\n", + "**4. Drop the redundant middle.** Once the tags look good, ask the agent:\n", + "\n", + "> *Discard train samples that have no \"easy_ex\" and no \"hard_ex\" tag.*\n", + "\n", + "The training set collapses from **~60k to roughly 3–4k** samples — only the informative extremes remain.\n", + "\n", + "**5. Fine-tune on the curated set.** **Resume** training (from the UI, or by re-running the training cell) and let it run for about **6,500 steps** on this much smaller, higher-signal dataset.\n", + "\n", + "**6. Evaluate.** Trigger a full evaluation from the UI: **right-click `resume` → `evaluate` → click `evaluate` → select `test_loader`**, then wait for it to finish.\n", + "\n", + "> **Result:** test accuracy climbs to roughly **0.95–0.99**, a large jump over the 0.86 baseline — reached by training on **~15× fewer** samples. Curating *which* data the model sees beat simply training on *more* of it.\n", + "\n", + "This is the core WeightsLab loop: **read the per-sample signals → curate the dataset → fine-tune → measure**, all without leaving the UI or restarting the run." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZnCx2Mh2FETl" + }, + "source": [ + "---\n", + "\n", + "
\n", + "Crafted by GrayboxTech - if WeightsLab helps you catch a bad label, drop us a star on GitHub.\n", + "
" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "include_colab_link": true, + "name": "wl-classification.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "1e96676b72ca44f399ae30822eb54501": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": "black", + "description_width": "" + } + }, + "257741e960664020981c3e0dc2a71a0b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "auto" + } + }, + "39cf7f4f60554bfd81859b9edeb7d41f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f35b21ef87c44a24ac9de9f3f7f58fa1", + "max": 100, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1e96676b72ca44f399ae30822eb54501", + "value": 100 + } + }, + "3a99ecc652df435d8d16c3d1ea4b6236": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6d29e91107f246a69f7ba1c9d3541747", + "max": 6000, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_ec736e6e5cf74dd7a644222821cb5899", + "value": 6000 + } + }, + "48d77b568b6d47579f27055722205bb7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c013e26a480400bafdfa45c36830b3c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "auto" + } + }, + "532b8013bc9549e692dbe8b20c99fe30": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5477fc60ab154377b690c8624b021c94", + "placeholder": "​", + "style": "IPY_MODEL_98af3369f706433f8dbb9be79d3988e2", + "value": " 6000/6000 [07:11<00:00, 45.13it/s, loss=1.774, test_acc=81.1%]" + } + }, + "5477fc60ab154377b690c8624b021c94": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f801ece792c416f90a86bde02d24514": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": "black", + "description_width": "" + } + }, + "6d29e91107f246a69f7ba1c9d3541747": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8658fa7fc5674b7bb2d6803336825cf2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_48d77b568b6d47579f27055722205bb7", + "placeholder": "​", + "style": "IPY_MODEL_f72491d19b5c4fb8824bb4962e1fa812", + "value": "Training: 100%" + } + }, + "8d93f084a8ae4ff0a442b8385a0febc7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_8658fa7fc5674b7bb2d6803336825cf2", + "IPY_MODEL_3a99ecc652df435d8d16c3d1ea4b6236", + "IPY_MODEL_532b8013bc9549e692dbe8b20c99fe30" + ], + "layout": "IPY_MODEL_c68d2e0fbc504ba8bf3d057b72b52293" + } + }, + "92834b1a78d1439181f9026917f4882d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": "black", + "description_width": "" + } + }, + "98af3369f706433f8dbb9be79d3988e2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a67abfb4e9a441bdb92f5ee92593aa5c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_257741e960664020981c3e0dc2a71a0b", + "max": 100, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_5f801ece792c416f90a86bde02d24514", + "value": 100 + } + }, + "c68d2e0fbc504ba8bf3d057b72b52293": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d0e534930f564af3b6fc47ff63286bd3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4c013e26a480400bafdfa45c36830b3c", + "max": 100, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_92834b1a78d1439181f9026917f4882d", + "value": 100 + } + }, + "ec736e6e5cf74dd7a644222821cb5899": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "f35b21ef87c44a24ac9de9f3f7f58fa1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "auto" + } + }, + "f72491d19b5c4fb8824bb4962e1fa812": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/PyTorch/wl-clustering.ipynb b/weightslab/examples/Notebooks/PyTorch/wl-clustering.ipynb new file mode 100644 index 00000000..4e5e1e9c --- /dev/null +++ b/weightslab/examples/Notebooks/PyTorch/wl-clustering.ipynb @@ -0,0 +1,5272 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "WvuZDK7iF391" + }, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Welcome to the WeightsLab Face Recognition (triplet loss) notebook! WeightsLab traces training signals back to the exact samples producing them. Browse the Docs for details.
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8SrJYFpJF392" + }, + "source": [ + "# Face Recognition (triplet loss) with WeightsLab\n", + "\n", + "Trains a ResNet-18 embedding head with online batch-hard **triplet loss** and tracks verification / retrieval / similarity signals per sample.\n", + "\n", + "> Data: Defaults to the **Olivetti Faces** dataset (via scikit-learn) which works **offline** - no download needed. Switch to LFW or a folder dataset in the config.\n", + "\n", + "This notebook is fully **self-contained**: the `face/` helper modules and the training loop\n", + "are inlined below, so there is no repo to clone and no `main.py` to shell out to. Run the\n", + "cells top to bottom." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NEWaY0IuF393" + }, + "source": [ + "## Setup\n", + "\n", + "On Colab, pick a **T4 GPU** runtime (`Runtime -> Change runtime type -> T4 GPU`)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v2Va8BtGF393" + }, + "source": [ + "\"PyPI\n", + "\"PyPI\n", + "\"PyPI" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "WaMERj7lF394", + "outputId": "19afa3e0-9dbc-4280-8ebf-03388009d2b4", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Looking in indexes: https://test.pypi.org/simple/, https://pypi.org/simple/\n", + "Collecting weightslab==1.3.3.dev7\n", + " Downloading https://test-files.pythonhosted.org/packages/fd/50/2281a781f96c471a8e8ec9f7e15cb6b63bcae66b362474f00f1f29e75428/weightslab-1.3.3.dev7-py3-none-any.whl.metadata (15 kB)\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (3.10.2)\n", + "Collecting torch<=2.9,>=2.1 (from weightslab==1.3.3.dev7)\n", + " Downloading torch-2.9.0-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (30 kB)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (0.26.0+cpu)\n", + "Collecting torchmetrics>=1.9 (from weightslab==1.3.3.dev7)\n", + " Downloading torchmetrics-1.9.0-py3-none-any.whl.metadata (23 kB)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (0.21)\n", + "Collecting onnx<=1.20,>=1.15 (from weightslab==1.3.3.dev7)\n", + " Downloading onnx-1.20.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (8.4 kB)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.4.8)\n", + "Collecting langchain-ollama<2,>=0.2 (from weightslab==1.3.3.dev7)\n", + " Downloading langchain_ollama-1.1.0-py3-none-any.whl.metadata (3.0 kB)\n", + "Collecting langchain-openai<2,>=0.2 (from weightslab==1.3.3.dev7)\n", + " Downloading langchain_openai-1.3.5-py3-none-any.whl.metadata (3.4 kB)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev7) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.16.2)\n", + "Collecting ollama<1.0.0,>=0.6.1 (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev7)\n", + " Downloading ollama-0.6.2-py3-none-any.whl.metadata (5.8 kB)\n", + "Collecting langchain-core<2,>=0.3 (from weightslab==1.3.3.dev7)\n", + " Downloading langchain_core-1.4.9-py3-none-any.whl.metadata (4.7 kB)\n", + "Collecting openai<3.0.0,>=2.45.0 (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev7)\n", + " Downloading openai-2.45.0-py3-none-any.whl.metadata (34 kB)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev7) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev7) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev7) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev7) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev7) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev7) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev7) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev7) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev7) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev7) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev7) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (2025.3.0)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.8.93 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl.metadata (1.7 kB)\n", + "Collecting nvidia-cuda-runtime-cu12==12.8.90 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.7 kB)\n", + "Collecting nvidia-cuda-cupti-cu12==12.8.90 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.7 kB)\n", + "Collecting nvidia-cudnn-cu12==9.10.2.21 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl.metadata (1.8 kB)\n", + "Collecting nvidia-cublas-cu12==12.8.4.1 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl.metadata (1.7 kB)\n", + "Collecting nvidia-cufft-cu12==11.3.3.83 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.7 kB)\n", + "Collecting nvidia-curand-cu12==10.3.9.90 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl.metadata (1.7 kB)\n", + "Collecting nvidia-cusolver-cu12==11.7.3.90 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl.metadata (1.8 kB)\n", + "Collecting nvidia-cusparse-cu12==12.5.8.93 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.8 kB)\n", + "Collecting nvidia-cusparselt-cu12==0.7.1 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl.metadata (7.0 kB)\n", + "Collecting nvidia-nccl-cu12==2.27.5 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (2.0 kB)\n", + "Collecting nvidia-nvshmem-cu12==3.3.20 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (2.1 kB)\n", + "Collecting nvidia-nvtx-cu12==12.8.90 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.8 kB)\n", + "Collecting nvidia-nvjitlink-cu12==12.8.93 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl.metadata (1.7 kB)\n", + "Collecting nvidia-cufile-cu12==1.13.1.3 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.7 kB)\n", + "Collecting triton==3.5.0 (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7)\n", + " Downloading triton-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (1.7 kB)\n", + "Collecting lightning-utilities>=0.15.3 (from torchmetrics>=1.9->weightslab==1.3.3.dev7)\n", + " Downloading lightning_utilities-0.15.3-py3-none-any.whl.metadata (5.5 kB)\n", + "INFO: pip is looking at multiple versions of torchvision to determine which version is compatible with other requirements. This could take a while.\n", + "Collecting torchvision<1,>=0.16 (from weightslab==1.3.3.dev7)\n", + " Downloading torchvision-0.28.0-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (5.6 kB)\n", + " Downloading torchvision-0.27.1-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (5.5 kB)\n", + " Downloading torchvision-0.27.0-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (5.5 kB)\n", + " Downloading torchvision-0.26.0-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (5.5 kB)\n", + " Downloading torchvision-0.25.0-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (5.4 kB)\n", + " Downloading torchvision-0.24.1-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (5.9 kB)\n", + " Downloading torchvision-0.24.0-cp312-cp312-manylinux_2_28_x86_64.whl.metadata (5.9 kB)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev7) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev7) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev7) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.0.0)\n", + "Downloading https://test-files.pythonhosted.org/packages/fd/50/2281a781f96c471a8e8ec9f7e15cb6b63bcae66b362474f00f1f29e75428/weightslab-1.3.3.dev7-py3-none-any.whl (2.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.8/2.8 MB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading langchain_ollama-1.1.0-py3-none-any.whl (31 kB)\n", + "Downloading langchain_openai-1.3.5-py3-none-any.whl (121 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading langchain_core-1.4.9-py3-none-any.whl (558 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m558.3/558.3 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading onnx-1.20.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (18.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.1/18.1 MB\u001b[0m \u001b[31m67.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading torch-2.9.0-cp312-cp312-manylinux_2_28_x86_64.whl (899.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m899.7/899.7 MB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl (594.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m594.3/594.3 MB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (10.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.2/10.2 MB\u001b[0m \u001b[31m96.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl (88.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m88.0/88.0 MB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (954 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m954.8/954.8 kB\u001b[0m \u001b[31m64.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl (706.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m706.8/706.8 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (193.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m193.1/193.1 MB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (1.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m76.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl (63.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.6/63.6 MB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl (267.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m267.5/267.5 MB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (288.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m288.2/288.2 MB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl (287.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m287.2/287.2 MB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (322.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m322.3/322.3 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl (39.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m39.3/39.3 MB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (124.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.7/124.7 MB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (89 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.0/90.0 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading triton-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (170.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m170.5/170.5 MB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading torchmetrics-1.9.0-py3-none-any.whl (983 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m983.4/983.4 kB\u001b[0m \u001b[31m58.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading torchvision-0.24.0-cp312-cp312-manylinux_2_28_x86_64.whl (8.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.1/8.1 MB\u001b[0m \u001b[31m82.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading lightning_utilities-0.15.3-py3-none-any.whl (31 kB)\n", + "Downloading ollama-0.6.2-py3-none-any.whl (15 kB)\n", + "Downloading openai-2.45.0-py3-none-any.whl (1.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m64.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: nvidia-cusparselt-cu12, triton, nvidia-nvtx-cu12, nvidia-nvshmem-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufile-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, lightning-utilities, onnx, nvidia-cusparse-cu12, nvidia-cufft-cu12, nvidia-cudnn-cu12, openai, ollama, nvidia-cusolver-cu12, torch, langchain-core, torchvision, torchmetrics, langchain-openai, langchain-ollama, weightslab\n", + " Attempting uninstall: nvidia-nccl-cu12\n", + " Found existing installation: nvidia-nccl-cu12 2.30.7\n", + " Uninstalling nvidia-nccl-cu12-2.30.7:\n", + " Successfully uninstalled nvidia-nccl-cu12-2.30.7\n", + " Attempting uninstall: openai\n", + " Found existing installation: openai 2.43.0\n", + " Uninstalling openai-2.43.0:\n", + " Successfully uninstalled openai-2.43.0\n", + " Attempting uninstall: torch\n", + " Found existing installation: torch 2.11.0+cpu\n", + " Uninstalling torch-2.11.0+cpu:\n", + " Successfully uninstalled torch-2.11.0+cpu\n", + " Attempting uninstall: langchain-core\n", + " Found existing installation: langchain-core 1.4.8\n", + " Uninstalling langchain-core-1.4.8:\n", + " Successfully uninstalled langchain-core-1.4.8\n", + " Attempting uninstall: torchvision\n", + " Found existing installation: torchvision 0.26.0+cpu\n", + " Uninstalling torchvision-0.26.0+cpu:\n", + " Successfully uninstalled torchvision-0.26.0+cpu\n", + "Successfully installed langchain-core-1.4.9 langchain-ollama-1.1.0 langchain-openai-1.3.5 lightning-utilities-0.15.3 nvidia-cublas-cu12-12.8.4.1 nvidia-cuda-cupti-cu12-12.8.90 nvidia-cuda-nvrtc-cu12-12.8.93 nvidia-cuda-runtime-cu12-12.8.90 nvidia-cudnn-cu12-9.10.2.21 nvidia-cufft-cu12-11.3.3.83 nvidia-cufile-cu12-1.13.1.3 nvidia-curand-cu12-10.3.9.90 nvidia-cusolver-cu12-11.7.3.90 nvidia-cusparse-cu12-12.5.8.93 nvidia-cusparselt-cu12-0.7.1 nvidia-nccl-cu12-2.27.5 nvidia-nvjitlink-cu12-12.8.93 nvidia-nvshmem-cu12-3.3.20 nvidia-nvtx-cu12-12.8.90 ollama-0.6.2 onnx-1.20.0 openai-2.45.0 torch-2.9.0 torchmetrics-1.9.0 torchvision-0.24.0 triton-3.5.0 weightslab-1.3.3.dev7\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install -q weightslab\n", + "%pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev7\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wJTexBG4F395" + }, + "source": [ + "## 1. Imports\n", + "\n", + "`weightslab` is imported as `wl`. The two `guard_*_context` managers scope a block as\n", + "training vs. evaluation so signals are attributed to the right phase. We also detect the\n", + "compute `device`." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "kYbxEdSMF395", + "outputId": "37ab580f-ae16-46ac-e394-9866f76469c7", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:47:42.131 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmpmfs9yakr/weightslab_logs/weightslab_20260716_084742.log\n", + "16/07/2026-08:47:42.132 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-08:47:42.134 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev7 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-08:47:42.137 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n", + "Using device: cpu\n" + ] + } + ], + "source": [ + "import logging\n", + "import os\n", + "import tempfile\n", + "\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from torch.utils.data import Dataset\n", + "from torchvision import models\n", + "import torchvision.transforms as T\n", + "\n", + "from typing import Any, Dict, List, Optional, Tuple\n", + "\n", + "import weightslab as wl\n", + "from weightslab.components.global_monitoring import (\n", + " guard_training_context,\n", + " guard_testing_context,\n", + ")\n", + "\n", + "logging.basicConfig(level=logging.ERROR)\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "print(f\"Using device: {device}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9nIr37kPF396" + }, + "source": [ + "## 2. Face helper modules (inlined)\n", + "\n", + "These are the `face/` package modules from the original example, inlined here so the\n", + "notebook is self-contained. Only the intra-package `from face. ...` imports were removed -\n", + "every class and function is otherwise unchanged. Order follows the dependencies:\n", + "**utils -> data -> model -> signals**." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "94QCaUYjF396" + }, + "outputs": [], + "source": [ + "# ===== face/utils.py =====\n", + "\"\"\"\n", + "Utility functions for face recognition training.\n", + "- Pairwise distance computation\n", + "- Online batch-hard triplet mining\n", + "- Verification and Rank-1 evaluation helpers\n", + "- Similarity grouping signals for clustering-oriented analysis\n", + "\"\"\"\n", + "\n", + "import numpy as np\n", + "import torch\n", + "\n", + "from typing import Tuple, Dict, List, Any\n", + "\n", + "\n", + "# ============================================================\n", + "# Distance helpers\n", + "# ============================================================\n", + "\n", + "def pairwise_distances(embeddings: torch.Tensor, squared: bool = False) -> torch.Tensor:\n", + " \"\"\"Compute pairwise L2 distance matrix for a batch of embeddings.\n", + "\n", + " Args:\n", + " embeddings: (B, D) tensor\n", + " squared: return squared L2 distances when True\n", + "\n", + " Returns:\n", + " (B, B) distance matrix\n", + " \"\"\"\n", + " dot = torch.matmul(embeddings, embeddings.t())\n", + " sq_norms = torch.diagonal(dot) # (B,)\n", + " distances = sq_norms.unsqueeze(0) - 2.0 * dot + sq_norms.unsqueeze(1)\n", + " distances = distances.clamp(min=0.0)\n", + "\n", + " if not squared:\n", + " # Avoid NaN gradients at exactly 0\n", + " mask = (distances == 0.0).float()\n", + " distances = distances + mask * 1e-16\n", + " distances = torch.sqrt(distances)\n", + " distances = distances * (1.0 - mask)\n", + "\n", + " return distances\n", + "\n", + "\n", + "# ============================================================\n", + "# Triplet mining\n", + "# ============================================================\n", + "\n", + "def mine_batch_hard(\n", + " embeddings: torch.Tensor,\n", + " labels: torch.Tensor,\n", + " squared: bool = False,\n", + ") -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n", + " \"\"\"Online batch-hard triplet mining (Hermans et al., 2017).\n", + "\n", + " For each anchor selects:\n", + " - Hardest positive : same-class sample with the **largest** distance\n", + " - Hardest negative : different-class sample with the **smallest** distance\n", + "\n", + " Args:\n", + " embeddings: (B, D) detached from graph during mining\n", + " labels: (B,) integer class ids\n", + " squared: use squared L2 distances for mining\n", + "\n", + " Returns:\n", + " anc_idx, pos_idx, neg_idx 1-D LongTensors; only valid anchors\n", + " (those that have at least one positive peer) are included.\n", + " \"\"\"\n", + " dist_mat = pairwise_distances(embeddings.detach(), squared=squared)\n", + " B = labels.shape[0]\n", + " device = labels.device\n", + "\n", + " same = labels.unsqueeze(0) == labels.unsqueeze(1) # (B, B)\n", + " diff = ~same\n", + " eye = torch.eye(B, dtype=torch.bool, device=device)\n", + "\n", + " # Hardest positive (exclude self)\n", + " pos_mask = same & ~eye\n", + " pos_dist = dist_mat * pos_mask.float()\n", + " _, pos_idx = pos_dist.max(dim=1)\n", + "\n", + " # Hardest negative (mask invalid positions with large value)\n", + " neg_dist = dist_mat + (~diff).float() * 1e9\n", + " _, neg_idx = neg_dist.min(dim=1)\n", + "\n", + " # Keep only anchors that have at least one positive peer in the batch\n", + " valid = pos_mask.any(dim=1)\n", + " anc_idx = torch.where(valid)[0]\n", + "\n", + " return anc_idx, pos_idx[anc_idx], neg_idx[anc_idx]\n", + "\n", + "\n", + "# ============================================================\n", + "# Numpy-level evaluation helpers\n", + "# ============================================================\n", + "\n", + "def compute_verification_metrics(\n", + " embeddings: np.ndarray,\n", + " labels: np.ndarray,\n", + " thresholds: np.ndarray | None = None,\n", + ") -> dict:\n", + " \"\"\"Pairwise verification accuracy over a threshold grid.\"\"\"\n", + " if thresholds is None:\n", + " thresholds = np.linspace(0.0, 2.0, 100)\n", + "\n", + " n = len(embeddings)\n", + "\n", + " # Pairwise L2 distances\n", + " dot = embeddings @ embeddings.T # (N, N)\n", + " sq = np.sum(embeddings ** 2, axis=1)\n", + " dist_mat = (sq[:, None] - 2.0 * dot + sq[None, :]).clip(min=0.0)\n", + " dist_mat = np.sqrt(dist_mat.clip(min=1e-16)) * (dist_mat != 0.0)\n", + "\n", + " same_pair = labels[:, None] == labels[None, :] # (N, N)\n", + "\n", + " # Upper triangle only (avoid double-counting)\n", + " iu = np.triu_indices(n, k=1)\n", + " dist_pairs = dist_mat[iu]\n", + " is_same_pair = same_pair[iu]\n", + "\n", + " n_same = int(is_same_pair.sum())\n", + " n_diff = int((~is_same_pair).sum())\n", + "\n", + " best_acc = 0.0\n", + " best_threshold = float(thresholds[len(thresholds) // 2])\n", + " best_far = 0.0\n", + " best_frr = 0.0\n", + "\n", + " for thr in thresholds:\n", + " pred_same = dist_pairs <= thr\n", + " tp = int((pred_same & is_same_pair).sum())\n", + " fp = int((pred_same & ~is_same_pair).sum())\n", + " fn = int((~pred_same & is_same_pair).sum())\n", + " tn = int((~pred_same & ~is_same_pair).sum())\n", + "\n", + " total = n_same + n_diff\n", + " acc = (tp + tn) / total if total > 0 else 0.0\n", + " far = fp / n_diff if n_diff > 0 else 0.0\n", + " frr = fn / n_same if n_same > 0 else 0.0\n", + "\n", + " if acc > best_acc:\n", + " best_acc = acc\n", + " best_threshold = float(thr)\n", + " best_far = far\n", + " best_frr = frr\n", + "\n", + " return {\n", + " \"verification_accuracy\": float(best_acc),\n", + " \"best_threshold\": float(best_threshold),\n", + " \"far\": float(best_far),\n", + " \"frr\": float(best_frr),\n", + " }\n", + "\n", + "\n", + "def compute_rank1_accuracy(embeddings: np.ndarray, labels: np.ndarray) -> float:\n", + " \"\"\"1-NN Rank-1 retrieval accuracy (leave-one-out).\"\"\"\n", + " n = len(embeddings)\n", + " dot = embeddings @ embeddings.T\n", + " sq = np.sum(embeddings ** 2, axis=1)\n", + " dist_mat = (sq[:, None] - 2.0 * dot + sq[None, :]).clip(min=0.0)\n", + "\n", + " # Exclude self\n", + " np.fill_diagonal(dist_mat, 1e9)\n", + " nn_idx = dist_mat.argmin(axis=1)\n", + "\n", + " correct = int(np.sum(labels[nn_idx] == labels))\n", + " return correct / n\n", + "\n", + "\n", + "def compute_similarity_grouping(\n", + " embeddings: np.ndarray,\n", + " uids: List[str],\n", + " cluster_eps: float = 0.6,\n", + " cluster_min_samples: int = 2,\n", + ") -> Dict[str, Any]:\n", + " \"\"\"Build per-sample similarity grouping signals for test-set clustering.\n", + "\n", + " Returns per-sample arrays that can be written as WeightsLAB signals:\n", + " - cluster_id: DBSCAN cluster index (-1 for noise)\n", + " - cluster_size: size of assigned cluster (1 for noise)\n", + " - nn1_uid: nearest-neighbor UID in the evaluated set\n", + " - nn1_distance: nearest-neighbor L2 distance\n", + " \"\"\"\n", + " n = len(embeddings)\n", + " if n == 0:\n", + " return {\n", + " \"cluster_id\": np.array([], dtype=np.int32),\n", + " \"cluster_size\": np.array([], dtype=np.int32),\n", + " \"nn1_uid\": [],\n", + " \"nn1_distance\": np.array([], dtype=np.float32),\n", + " \"num_clusters\": 0.0,\n", + " \"noise_ratio\": 0.0,\n", + " \"mean_nn1_distance\": float(\"nan\"),\n", + " }\n", + "\n", + " # Pairwise distances for NN lookup\n", + " dot = embeddings @ embeddings.T\n", + " sq = np.sum(embeddings ** 2, axis=1)\n", + " dist_mat = (sq[:, None] - 2.0 * dot + sq[None, :]).clip(min=0.0)\n", + " dist_mat = np.sqrt(dist_mat.clip(min=1e-16))\n", + "\n", + " if n == 1:\n", + " nn_idx = np.array([0], dtype=np.int64)\n", + " nn_dist = np.array([0.0], dtype=np.float32)\n", + " else:\n", + " np.fill_diagonal(dist_mat, 1e9)\n", + " nn_idx = dist_mat.argmin(axis=1)\n", + " nn_dist = dist_mat[np.arange(n), nn_idx].astype(np.float32)\n", + "\n", + " nn_uid = [uids[int(i)] for i in nn_idx.tolist()]\n", + "\n", + " # Unsupervised clustering on embeddings\n", + " try:\n", + " from sklearn.cluster import DBSCAN\n", + "\n", + " labels = DBSCAN(eps=cluster_eps, min_samples=cluster_min_samples, metric=\"euclidean\").fit_predict(embeddings)\n", + " except Exception:\n", + " labels = np.full((n,), -1, dtype=np.int32)\n", + "\n", + " labels = labels.astype(np.int32)\n", + " valid_labels = labels[labels >= 0]\n", + " num_clusters = int(len(np.unique(valid_labels))) if valid_labels.size > 0 else 0\n", + "\n", + " counts = {}\n", + " if valid_labels.size > 0:\n", + " uniq, cnt = np.unique(valid_labels, return_counts=True)\n", + " counts = {int(k): int(v) for k, v in zip(uniq.tolist(), cnt.tolist())}\n", + "\n", + " cluster_size = np.array([counts.get(int(c), 1) if int(c) >= 0 else 1 for c in labels], dtype=np.int32)\n", + " noise_ratio = float(np.mean(labels < 0))\n", + "\n", + " return {\n", + " \"cluster_id\": labels,\n", + " \"cluster_size\": cluster_size,\n", + " \"nn1_uid\": nn_uid,\n", + " \"nn1_distance\": nn_dist,\n", + " \"num_clusters\": float(num_clusters),\n", + " \"noise_ratio\": noise_ratio,\n", + " \"mean_nn1_distance\": float(nn_dist.mean()) if len(nn_dist) > 0 else float(\"nan\"),\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "P6rEQuj6F397" + }, + "outputs": [], + "source": [ + "# ===== face/data.py =====\n", + "\"\"\"\n", + "FaceDataset — toy face recognition dataset.\n", + "\n", + "Supported back-ends\n", + "-------------------\n", + "\"olivetti\" Olivetti Faces from sklearn (40 identities, 400 images, 64×64 grey).\n", + " Self-contained; requires only scikit-learn. Good default toy set.\n", + "\"lfw\" Labeled Faces in the Wild (LFW) via torchvision. Downloaded on\n", + " first use to *root*. Much larger but realistic.\n", + "\"folder\" Generic ImageFolder layout: root/{split}/{class_name}/*.jpg\n", + "\n", + "Every sample is returned as:\n", + " (image_tensor: Tensor[C,H,W],\n", + " uid: str,\n", + " label: int,\n", + " metadata: dict)\n", + "\n", + "These map directly onto the (data, uid, target, metadata) convention used\n", + "throughout the WeightsLAB kitchen examples so the same training loop works\n", + "without modification.\n", + "\"\"\"\n", + "\n", + "import os\n", + "import logging\n", + "\n", + "import numpy as np\n", + "import torch\n", + "from torch.utils.data import Dataset\n", + "from typing import Dict, Optional, Tuple\n", + "\n", + "import torchvision.transforms as T\n", + "\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "\n", + "# ============================================================\n", + "# Dataset\n", + "# ============================================================\n", + "\n", + "class FaceDataset(Dataset):\n", + " \"\"\"Unified face recognition dataset wrapper.\n", + "\n", + " Args:\n", + " root: Download / data root (only used for lfw / folder).\n", + " dataset_type: One of \"olivetti\", \"lfw\", \"folder\".\n", + " split: \"train\" or \"test\" (ignored for pre-split sources).\n", + " image_size: Spatial size; images are resized to (image_size, image_size).\n", + " train_ratio: Fraction of per-class samples used for training\n", + " (Olivetti only).\n", + " min_images_per_class: Classes with fewer samples are discarded.\n", + " transform: Optional torchvision transform; defaults to\n", + " Resize → ToTensor → Normalize([0.5], [0.5]).\n", + " seed: RNG seed for reproducible train/test splits.\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " root: str = \".\",\n", + " dataset_type: str = \"olivetti\",\n", + " split: str = \"train\",\n", + " image_size: int = 64,\n", + " train_ratio: float = 0.8,\n", + " min_images_per_class: int = 2,\n", + " transform=None,\n", + " seed: int = 42,\n", + " ):\n", + " self.dataset_type = dataset_type\n", + " self.split = split\n", + " self.image_size = image_size\n", + " self.transform = transform or self._default_transform(image_size)\n", + "\n", + " # These are populated by each loader\n", + " self.images: Optional[np.ndarray] = None # (N, H, W) float [0,1] — Olivetti only\n", + " self.img_paths: Optional[np.ndarray] = None\n", + " self.labels: np.ndarray = np.array([], dtype=np.int64)\n", + " self.num_classes: int = 0\n", + "\n", + " if dataset_type == \"olivetti\":\n", + " self._load_olivetti(train_ratio, min_images_per_class, seed)\n", + " elif dataset_type == \"lfw\":\n", + " self._load_lfw(root, min_images_per_class, split)\n", + " elif dataset_type == \"folder\":\n", + " self._load_folder(root, split)\n", + " else:\n", + " raise ValueError(\n", + " f\"Unknown dataset_type {dataset_type!r}. \"\n", + " \"Choose from 'olivetti', 'lfw', 'folder'.\"\n", + " )\n", + "\n", + " logger.info(\n", + " f\"FaceDataset [{dataset_type} / {split}] \"\n", + " f\"| samples={len(self)} | classes={self.num_classes}\"\n", + " )\n", + "\n", + " # ----------------------------------------------------------\n", + " # Loaders\n", + " # ----------------------------------------------------------\n", + "\n", + " @staticmethod\n", + " def _default_transform(image_size: int):\n", + " return T.Compose([\n", + " T.Resize((image_size, image_size)),\n", + " T.ToTensor(),\n", + " T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),\n", + " ])\n", + "\n", + " def _load_olivetti(self, train_ratio: float, min_images: int, seed: int):\n", + " \"\"\"Load and split the Olivetti Faces dataset (sklearn).\"\"\"\n", + " from sklearn.datasets import fetch_olivetti_faces\n", + "\n", + " data = fetch_olivetti_faces(shuffle=True, random_state=seed)\n", + " images = data.images # (400, 64, 64) float [0,1]\n", + " labels = data.target.astype(np.int64)\n", + "\n", + " # Drop classes with insufficient samples\n", + " unique, counts = np.unique(labels, return_counts=True)\n", + " valid_classes = unique[counts >= min_images]\n", + " mask = np.isin(labels, valid_classes)\n", + " images, labels = images[mask], labels[mask]\n", + "\n", + " # Remap labels to a contiguous 0…N-1 range\n", + " mapping = {int(c): i for i, c in enumerate(sorted(valid_classes.tolist()))}\n", + " labels = np.array([mapping[int(l)] for l in labels], dtype=np.int64)\n", + "\n", + " # Per-class stratified train/test split\n", + " rng = np.random.RandomState(seed)\n", + " train_idx, test_idx = [], []\n", + " for cls in np.unique(labels):\n", + " idx = np.where(labels == cls)[0]\n", + " n_train = max(1, int(len(idx) * train_ratio))\n", + " perm = rng.permutation(len(idx))\n", + " train_idx.extend(idx[perm[:n_train]].tolist())\n", + " test_idx.extend(idx[perm[n_train:]].tolist())\n", + "\n", + " indices = train_idx if self.split == \"train\" else test_idx\n", + " self.images = images[indices]\n", + " self.labels = labels[indices]\n", + " self.num_classes = len(mapping)\n", + "\n", + " def _load_lfw(self, root: str, min_images: int, split: str):\n", + " \"\"\"Load LFW People via torchvision (downloads on first call).\"\"\"\n", + " from torchvision.datasets import LFWPeople\n", + "\n", + " split_map = {\"train\": \"train\", \"test\": \"test\", \"val\": \"10fold\"}\n", + " lfw_split = split_map.get(split, \"train\")\n", + " ds = LFWPeople(root=root, split=lfw_split, download=True, transform=None)\n", + "\n", + " paths, lbls = zip(*ds.imgs)\n", + " lbls = np.array(lbls, dtype=np.int64)\n", + "\n", + " # Filter low-shot identities\n", + " unique, counts = np.unique(lbls, return_counts=True)\n", + " valid = set(unique[counts >= min_images].tolist())\n", + " mask = np.array([int(l) in valid for l in lbls])\n", + "\n", + " self.img_paths = np.array(paths)[mask]\n", + " lbls = lbls[mask]\n", + "\n", + " mapping = {int(c): i for i, c in enumerate(sorted(valid))}\n", + " self.labels = np.array([mapping[int(l)] for l in lbls], dtype=np.int64)\n", + " self.num_classes = len(mapping)\n", + "\n", + " def _load_folder(self, root: str, split: str):\n", + " \"\"\"Load from a torchvision ImageFolder directory.\"\"\"\n", + " from torchvision.datasets import ImageFolder\n", + "\n", + " split_dir = os.path.join(root, split)\n", + " ds = ImageFolder(split_dir)\n", + " paths, lbls = zip(*ds.imgs)\n", + " self.img_paths = list(paths)\n", + " self.labels = np.array(lbls, dtype=np.int64)\n", + " self.num_classes = len(ds.classes)\n", + "\n", + " # ----------------------------------------------------------\n", + " # Dataset interface\n", + " # ----------------------------------------------------------\n", + "\n", + " def __len__(self) -> int:\n", + " return len(self.labels)\n", + "\n", + " def __getitem__(self, idx: int) -> Tuple[torch.Tensor, str, int, Dict]:\n", + " \"\"\"Return (image_tensor, uid, label_id, metadata).\n", + "\n", + " The (data, uid, target, metadata) signature mirrors the WeightsLAB\n", + " kitchen convention used in the VLA training example.\n", + " \"\"\"\n", + " label = int(self.labels[idx])\n", + "\n", + " if self.dataset_type == \"olivetti\":\n", + " from PIL import Image as PILImage\n", + " img_np = self.images[idx] # (H, W) float [0,1]\n", + " img_pil = PILImage.fromarray(\n", + " (img_np * 255).astype(np.uint8), mode=\"L\"\n", + " ).convert(\"RGB\")\n", + " else:\n", + " from PIL import Image as PILImage\n", + " img_pil = PILImage.open(self.img_paths[idx]).convert(\"RGB\")\n", + "\n", + " image_tensor = self.transform(img_pil)\n", + "\n", + " uid = f\"{self.split}_cls{label:04d}_idx{idx:06d}\"\n", + " metadata = {\n", + " \"split\": self.split,\n", + " \"label_id\": label,\n", + " \"idx\": idx,\n", + " \"dataset_type\": self.dataset_type,\n", + " }\n", + " return image_tensor, uid, label, metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "0mFrPy3QF397" + }, + "outputs": [], + "source": [ + "# ===== face/model.py =====\n", + "\"\"\"\n", + "Face embedding model.\n", + "\n", + "Architecture\n", + "------------\n", + "Pretrained backbone (ResNet-18 / ResNet-50 / MobileNet-V3-Small)\n", + " ?\n", + "EmbeddingHead Linear ? BN ? ReLU ? Linear\n", + " ?\n", + "L2-normalised D-dimensional embedding\n", + "\n", + "The backbone is optionally frozen so that only the lightweight head is trained\n", + "(recommended toy-example setup). The combined graph is registered with\n", + "WeightsLAB for model tracking.\n", + "\n", + "Public interface\n", + "----------------\n", + "FaceEmbeddingModel.get_embeddings(images) ? normalised embeddings (B, D)\n", + "FaceEmbeddingModel.train_step(images, labels, ...) ? scalar loss float\n", + "\"\"\"\n", + "\n", + "import logging\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "\n", + "import weightslab as wl\n", + "\n", + "from torchvision import models\n", + "from typing import List\n", + "\n", + "\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "\n", + "# ============================================================\n", + "# Network modules\n", + "# ============================================================\n", + "\n", + "class EmbeddingHead(nn.Module):\n", + " \"\"\"Projection head: backbone_feature_dim ? embedding_dim (L2-normalised).\"\"\"\n", + "\n", + " def __init__(self, in_dim: int, hidden_dim: int, embedding_dim: int):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(in_dim, hidden_dim),\n", + " nn.BatchNorm1d(hidden_dim),\n", + " nn.ReLU(inplace=True),\n", + " nn.Linear(hidden_dim, embedding_dim),\n", + " )\n", + "\n", + " def forward(self, x: torch.Tensor) -> torch.Tensor:\n", + " return F.normalize(self.net(x), p=2, dim=-1)\n", + "\n", + "\n", + "class FaceEmbeddingNet(nn.Module):\n", + " \"\"\"Single nn.Module combining the frozen backbone and trainable head.\"\"\"\n", + "\n", + " def __init__(self, backbone: nn.Module, head: EmbeddingHead):\n", + " super().__init__()\n", + " self.backbone = backbone\n", + " self.head = head\n", + "\n", + " def forward(self, x: torch.Tensor) -> torch.Tensor:\n", + " features = self.backbone(x) # (B, feature_dim)\n", + " embeddings = self.head(features) # (B, embedding_dim), L2-normalised\n", + " return embeddings\n", + "\n", + "\n", + "class TripletLossWithMetadata(nn.Module):\n", + " \"\"\"Triplet loss wrapper that logs anchor-level triplet metadata in WeightsLAB.\n", + "\n", + " The anchor UID is used as batch ID. For each anchor we persist the selected\n", + " positive and negative UIDs into per-sample signal columns so they remain\n", + " attached to image/sample metadata in the ledger.\n", + " \"\"\"\n", + "\n", + " def __init__(self, margin: float = 0.3):\n", + " super().__init__()\n", + " self.margin = margin\n", + " self.criterion = nn.TripletMarginLoss(margin=margin, p=2, reduce=False)\n", + "\n", + " def forward(\n", + " self,\n", + " anchor_emb: torch.Tensor,\n", + " pos_emb: torch.Tensor,\n", + " neg_emb: torch.Tensor,\n", + " anchor_ids: List[str],\n", + " pos_ids: List[str],\n", + " neg_ids: List[str],\n", + " name: str = \"train\",\n", + " ) -> torch.Tensor:\n", + " loss = self.criterion(anchor_emb, pos_emb, neg_emb)\n", + "\n", + " wl.save_signals(\n", + " batch_ids=anchor_ids,\n", + " signals={f\"{name}/loss/triplet\": loss.detach().cpu().numpy()},\n", + " preds_raw=anchor_emb,\n", + " targets=neg_emb,\n", + " preds=None,\n", + " log=True,\n", + " )\n", + "\n", + " # Store pivot triplet composition as per-sample metadata-like signals.\n", + " wl.save_signals(\n", + " batch_ids=anchor_ids,\n", + " signals={\n", + " f\"{name}/meta/triplet_pos_uid\": pos_ids,\n", + " f\"{name}/meta/triplet_neg_uid\": neg_ids,\n", + " },\n", + " preds_raw=None,\n", + " targets=None,\n", + " preds=None,\n", + " log=False,\n", + " )\n", + " return loss\n", + "\n", + "\n", + "# ============================================================\n", + "# High-level model wrapper\n", + "# ============================================================\n", + "\n", + "class FaceEmbeddingModel:\n", + " \"\"\"Wrapper that manages the backbone + head, optimiser, and WeightsLAB tracking.\n", + "\n", + " Args:\n", + " backbone_name: \"resnet18\" | \"resnet50\" | \"mobilenet_v3_small\"\n", + " embedding_dim: Output embedding dimensionality (default 128).\n", + " head_hidden_dim: Hidden size of the projection MLP (default 256).\n", + " lr: Learning rate for AdamW (default 1e-3).\n", + " weight_decay: AdamW weight decay (default 1e-4).\n", + " freeze_backbone: When True, only the head's parameters receive\n", + " gradients ? recommended for quick toy runs.\n", + " device: \"cpu\", \"cuda\", or \"cuda:N\".\n", + " pretrained: Load ImageNet-pretrained weights for the backbone.\n", + " margin: Triplet margin (default 0.3).\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " backbone_name: str = \"resnet18\",\n", + " embedding_dim: int = 128,\n", + " head_hidden_dim: int = 256,\n", + " lr: float = 1e-3,\n", + " weight_decay: float = 1e-4,\n", + " freeze_backbone: bool = True,\n", + " device: str = \"cpu\",\n", + " pretrained: bool = True,\n", + " margin: float = 0.3,\n", + " ):\n", + " self.device = torch.device(device)\n", + " self.margin = margin\n", + " self.embedding_dim = embedding_dim\n", + "\n", + " # ---- Build backbone ----\n", + " weights = \"DEFAULT\" if pretrained else None\n", + " backbone_feature_dim = self._build_backbone(backbone_name, weights, freeze_backbone)\n", + "\n", + " # ---- Build head ----\n", + " head = EmbeddingHead(\n", + " in_dim=backbone_feature_dim,\n", + " hidden_dim=head_hidden_dim,\n", + " embedding_dim=embedding_dim,\n", + " )\n", + "\n", + " # ---- Compose & move to device ----\n", + " self.net = FaceEmbeddingNet(backbone=self._backbone, head=head).to(self.device)\n", + "\n", + " # ---- WeightsLAB graph tracking ----\n", + " self.net = wl.watch_or_edit(\n", + " self.net,\n", + " flag=\"model\",\n", + " compute_dependencies=False,\n", + " device=str(self.device),\n", + " )\n", + "\n", + " # ---- WeightsLAB tracked loss ----\n", + " self.triplet_loss_fn = TripletLossWithMetadata(margin=self.margin)\n", + " self.triplet_loss_fn.__name__ = \"tripletcriterion\"\n", + " watched_loss = wl.watch_or_edit(\n", + " self.triplet_loss_fn,\n", + " flag=\"loss\",\n", + " signal_name=\"triplet_with_metadata\",\n", + " log=False,\n", + " use_batch_ids_as_x=True,\n", + " use_batch_value_as_y=False,\n", + " )\n", + " if watched_loss is not None and hasattr(watched_loss, \"forward\"):\n", + " self.triplet_loss_fn = watched_loss\n", + "\n", + " # ---- Optimiser (head-only when backbone is frozen) ----\n", + " trainable = [p for p in self.net.parameters() if p.requires_grad]\n", + " self.optimizer = torch.optim.AdamW(\n", + " trainable, lr=lr, weight_decay=weight_decay\n", + " )\n", + "\n", + " n_trainable = sum(p.numel() for p in trainable)\n", + " logger.info(\n", + " f\"FaceEmbeddingModel | backbone={backbone_name} pretrained={pretrained} \"\n", + " f\"frozen={freeze_backbone} | emb_dim={embedding_dim} | \"\n", + " f\"trainable_params={n_trainable:,}\"\n", + " )\n", + " print(\n", + " f\" Backbone : {backbone_name} (pretrained={pretrained}, frozen={freeze_backbone})\\n\"\n", + " f\" Emb dim : {embedding_dim}\\n\"\n", + " f\" Head dim : {head_hidden_dim}\\n\"\n", + " f\" Trainable : {n_trainable:,} params\\n\"\n", + " f\" Device : {self.device}\"\n", + " )\n", + "\n", + " def _build_backbone(\n", + " self,\n", + " backbone_name: str,\n", + " weights,\n", + " freeze: bool,\n", + " ) -> int:\n", + " \"\"\"Instantiate backbone, strip its classifier, optionally freeze.\n", + "\n", + " Sets self._backbone and returns the feature dimensionality.\n", + " \"\"\"\n", + " if backbone_name == \"resnet18\":\n", + " base = models.resnet18(weights=weights)\n", + " feature_dim = 512\n", + " base.fc = nn.Identity()\n", + " elif backbone_name == \"resnet50\":\n", + " base = models.resnet50(weights=weights)\n", + " feature_dim = 2048\n", + " base.fc = nn.Identity()\n", + " elif backbone_name == \"mobilenet_v3_small\":\n", + " base = models.mobilenet_v3_small(weights=weights)\n", + " feature_dim = 576\n", + " base.classifier = nn.Identity()\n", + " else:\n", + " raise ValueError(\n", + " f\"Unsupported backbone {backbone_name!r}. \"\n", + " \"Choose from 'resnet18', 'resnet50', 'mobilenet_v3_small'.\"\n", + " )\n", + "\n", + " if freeze:\n", + " for param in base.parameters():\n", + " param.requires_grad_(False)\n", + " logger.info(\"Backbone frozen ? training head only.\")\n", + "\n", + " self._backbone = base\n", + " return feature_dim\n", + "\n", + " # ----------------------------------------------------------\n", + " # Inference\n", + " # ----------------------------------------------------------\n", + "\n", + " def get_embeddings(self, images: torch.Tensor) -> torch.Tensor:\n", + " \"\"\"Forward pass in eval mode; returns L2-normalised embeddings (B, D).\n", + "\n", + " Args:\n", + " images: (B, C, H, W) float tensor (GPU/CPU ? moved internally)\n", + "\n", + " Returns:\n", + " (B, D) embedding tensor on CPU\n", + " \"\"\"\n", + " self.net.eval()\n", + " with torch.no_grad():\n", + " emb = self.net(images.to(self.device))\n", + " return emb.cpu()\n", + "\n", + " # ----------------------------------------------------------\n", + " # Training step\n", + " # ----------------------------------------------------------\n", + "\n", + " def train_step(\n", + " self,\n", + " images: torch.Tensor,\n", + " labels: torch.Tensor,\n", + " batch_ids: List[str],\n", + " loss_name: str = \"triplet\",\n", + " ) -> float:\n", + " \"\"\"One gradient update using online batch-hard triplet mining.\n", + "\n", + " Args:\n", + " images: (B, C, H, W) float tensor\n", + " labels: (B,) long tensor of identity ids\n", + " batch_ids: list of sample UIDs for WeightsLAB signal logging\n", + " loss_name: \"triplet\" (contrastive support planned)\n", + "\n", + " Returns:\n", + " Scalar loss value (Python float)\n", + " \"\"\"\n", + " self.net.train()\n", + " self.optimizer.zero_grad()\n", + "\n", + " images = images.to(self.device)\n", + " labels = labels.to(self.device)\n", + "\n", + " embeddings = self.net(images) # (B, D)\n", + "\n", + " # Mine hardest triplets in the batch\n", + " anc_idx, pos_idx, neg_idx = mine_batch_hard(embeddings, labels)\n", + "\n", + " if anc_idx.numel() == 0:\n", + " logger.warning(\n", + " \"No valid triplets found in batch (all samples same class?); \"\n", + " \"skipping gradient step.\"\n", + " )\n", + " return 0.0\n", + "\n", + " anc_emb = embeddings[anc_idx]\n", + " pos_emb = embeddings[pos_idx]\n", + " neg_emb = embeddings[neg_idx]\n", + "\n", + " triplet_ids = [batch_ids[i] for i in anc_idx.tolist()]\n", + " pos_triplet_ids = [batch_ids[i] for i in pos_idx.tolist()]\n", + " neg_triplet_ids = [batch_ids[i] for i in neg_idx.tolist()]\n", + "\n", + " if loss_name == \"triplet\":\n", + " loss_by_sample = self.triplet_loss_fn(\n", + " anchor_emb=anc_emb,\n", + " pos_emb=pos_emb,\n", + " neg_emb=neg_emb,\n", + " anchor_ids=triplet_ids,\n", + " pos_ids=pos_triplet_ids,\n", + " neg_ids=neg_triplet_ids,\n", + " name=\"train\",\n", + " )\n", + " else:\n", + " raise ValueError(\n", + " f\"Unsupported loss {loss_name!r}. Use 'triplet'.\"\n", + " )\n", + " # Aggregate per-sample losses into a single scalar and backprop\n", + " loss = loss_by_sample.mean()\n", + " loss.backward()\n", + " torch.nn.utils.clip_grad_norm_(\n", + " [p for p in self.net.parameters() if p.requires_grad],\n", + " max_norm=1.0,\n", + " )\n", + " self.optimizer.step()\n", + "\n", + " return float(loss.item())" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "zH13fDb8F398" + }, + "outputs": [], + "source": [ + "# ===== face/signals.py =====\n", + "\"\"\"\n", + "Signals (losses and metrics) for face recognition training.\n", + "\n", + "Follows the WeightsLAB pattern: every helper calls wl.save_signals\n", + "immediately after computing the value so the platform captures it.\n", + "\n", + "Classes\n", + "-------\n", + "TripletLosses differentiable loss functions (return torch.Tensor)\n", + "FaceMetrics evaluation metrics and clustering-oriented test signals\n", + "\"\"\"\n", + "\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "\n", + "import weightslab as wl\n", + "\n", + "from typing import Dict\n", + "\n", + "\n", + "\n", + "# ============================================================\n", + "# Loss functions\n", + "# ============================================================\n", + "\n", + "class TripletLosses:\n", + " \"\"\"Differentiable loss functions for face embedding training.\"\"\"\n", + "\n", + " @staticmethod\n", + " def triplet_loss(\n", + " ids,\n", + " anchor_emb: torch.Tensor,\n", + " pos_emb: torch.Tensor,\n", + " neg_emb: torch.Tensor,\n", + " margin: float = 0.3,\n", + " name: str = \"train\",\n", + " ) -> torch.Tensor:\n", + " \"\"\"Standard triplet margin loss on pre-mined (a, p, n) embeddings.\"\"\"\n", + " loss = nn.TripletMarginLoss(margin=margin, p=2)(anchor_emb, pos_emb, neg_emb)\n", + " wl.save_signals(\n", + " batch_ids=ids,\n", + " signals={f\"{name}/loss/triplet\": loss.detach().cpu().item()},\n", + " preds_raw=anchor_emb,\n", + " targets=neg_emb,\n", + " preds=None,\n", + " log=True,\n", + " )\n", + " return loss\n", + "\n", + " @staticmethod\n", + " def contrastive_loss(\n", + " ids,\n", + " emb_a: torch.Tensor,\n", + " emb_b: torch.Tensor,\n", + " same_label: torch.Tensor,\n", + " margin: float = 1.0,\n", + " name: str = \"train\",\n", + " ) -> torch.Tensor:\n", + " \"\"\"Contrastive loss (Hadsell et al., 2006).\"\"\"\n", + " dist = nn.functional.pairwise_distance(emb_a, emb_b)\n", + " loss = (\n", + " same_label * dist.pow(2)\n", + " + (1.0 - same_label) * (margin - dist).clamp(min=0.0).pow(2)\n", + " ).mean()\n", + " wl.save_signals(\n", + " batch_ids=ids,\n", + " signals={f\"{name}/loss/contrastive\": loss.detach().cpu().item()},\n", + " preds_raw=emb_a,\n", + " targets=emb_b,\n", + " preds=None,\n", + " log=True,\n", + " )\n", + " return loss\n", + "\n", + "\n", + "# ============================================================\n", + "# Evaluation metrics\n", + "# ============================================================\n", + "\n", + "class FaceMetrics:\n", + " \"\"\"Evaluation metrics for face verification, retrieval, and grouping.\"\"\"\n", + "\n", + " @staticmethod\n", + " def verification_accuracy(\n", + " ids,\n", + " embeddings: np.ndarray,\n", + " labels: np.ndarray,\n", + " name: str = \"test\",\n", + " ) -> Dict[str, float]:\n", + " \"\"\"Pairwise verification accuracy with optimal threshold search.\"\"\"\n", + " result = compute_verification_metrics(embeddings, labels)\n", + " wl.save_signals(\n", + " batch_ids=ids,\n", + " signals={\n", + " f\"{name}/metric/verification_accuracy\": result[\"verification_accuracy\"],\n", + " f\"{name}/metric/far\": result[\"far\"],\n", + " f\"{name}/metric/frr\": result[\"frr\"],\n", + " f\"{name}/metric/best_threshold\": result[\"best_threshold\"],\n", + " },\n", + " preds_raw=None,\n", + " targets=None,\n", + " preds=None,\n", + " log=True,\n", + " )\n", + " return result\n", + "\n", + " @staticmethod\n", + " def rank1_accuracy(\n", + " ids,\n", + " embeddings: np.ndarray,\n", + " labels: np.ndarray,\n", + " name: str = \"test\",\n", + " ) -> float:\n", + " \"\"\"1-NN Rank-1 retrieval accuracy (leave-one-out).\"\"\"\n", + " acc = compute_rank1_accuracy(embeddings, labels)\n", + " wl.save_signals(\n", + " batch_ids=ids,\n", + " signals={f\"{name}/metric/rank1_accuracy\": acc},\n", + " preds_raw=None,\n", + " targets=None,\n", + " preds=None,\n", + " log=True,\n", + " )\n", + " return acc\n", + "\n", + " @staticmethod\n", + " def similarity_grouping_signals(\n", + " ids,\n", + " embeddings: np.ndarray,\n", + " name: str = \"test\",\n", + " cluster_eps: float = 0.6,\n", + " cluster_min_samples: int = 2,\n", + " ) -> Dict[str, float]:\n", + " \"\"\"Save per-sample grouping signals for later clustering/sorting in Studio.\"\"\"\n", + " grouping = compute_similarity_grouping(\n", + " embeddings=embeddings,\n", + " uids=list(ids),\n", + " cluster_eps=cluster_eps,\n", + " cluster_min_samples=cluster_min_samples,\n", + " )\n", + "\n", + " wl.save_signals(\n", + " batch_ids=ids,\n", + " signals={\n", + " f\"{name}/cluster/id\": grouping[\"cluster_id\"],\n", + " f\"{name}/cluster/size\": grouping[\"cluster_size\"],\n", + " f\"{name}/cluster/nn1_distance\": grouping[\"nn1_distance\"],\n", + " f\"{name}/cluster/nn1_uid\": grouping[\"nn1_uid\"],\n", + " },\n", + " preds_raw=None,\n", + " targets=None,\n", + " preds=None,\n", + " log=False,\n", + " )\n", + "\n", + " summary = {\n", + " \"num_clusters\": grouping[\"num_clusters\"],\n", + " \"noise_ratio\": grouping[\"noise_ratio\"],\n", + " \"mean_nn1_distance\": grouping[\"mean_nn1_distance\"],\n", + " }\n", + " wl.save_signals(\n", + " batch_ids=ids,\n", + " signals={\n", + " f\"{name}/cluster/num_clusters\": summary[\"num_clusters\"],\n", + " f\"{name}/cluster/noise_ratio\": summary[\"noise_ratio\"],\n", + " f\"{name}/cluster/mean_nn1_distance\": summary[\"mean_nn1_distance\"],\n", + " },\n", + " preds_raw=None,\n", + " targets=None,\n", + " preds=None,\n", + " log=True,\n", + " )\n", + " return summary\n", + "\n", + " @staticmethod\n", + " def compute_all_metrics(\n", + " ids,\n", + " embeddings: np.ndarray,\n", + " labels: np.ndarray,\n", + " name: str = \"test\",\n", + " ) -> Dict[str, float]:\n", + " \"\"\"Run all metrics and return a merged dict.\"\"\"\n", + " verif = FaceMetrics.verification_accuracy(ids, embeddings, labels, name=name)\n", + " rank1 = FaceMetrics.rank1_accuracy(ids, embeddings, labels, name=name)\n", + " grouping = FaceMetrics.similarity_grouping_signals(ids, embeddings, name=name)\n", + " return {**verif, \"rank1_accuracy\": rank1, **grouping}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IiE3R5OrF398" + }, + "source": [ + "## 3. Configuration\n", + "\n", + "Every tunable lives in one `config` dict (the inlined `config.yaml`). Wrapping it with\n", + "`flag=\"hyperparameters\"` lets the Studio UI read and live-edit these values while training." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "7f3X3gveF398", + "outputId": "ccbe3f77-aefc-4d7d-d4be-4eea01f86dc9", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:47:42.439 INFO:weightslab.src:watch_or_edit: LoggerQueue for experiment history has been initialized and registered.\n", + "16/07/2026-08:47:42.442 INFO:weightslab.components.checkpoint_manager:__init__: CheckpointManager initialized at /tmp/tmpu_lry8gs\n", + "16/07/2026-08:47:42.443 INFO:weightslab.src:watch_or_edit: Registered new checkpoint manager in ledger\n", + "16/07/2026-08:47:42.444 INFO:root:set_log_directory: Log file moved from /tmp/tmpmfs9yakr/weightslab_logs/weightslab_20260716_084742.log to /tmp/tmpu_lry8gs/weightslab_20260716_084742.log\n", + "16/07/2026-08:47:42.445 INFO:root:set_log_directory: Log directory updated to: /tmp/tmpu_lry8gs\n", + "16/07/2026-08:47:42.446 INFO:root:set_log_directory: Log file: /tmp/tmpu_lry8gs/weightslab_20260716_084742.log\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "{'experiment_name': 'face_triplet_training',\n", + " 'device': 'auto',\n", + " 'eval_full_to_train_steps_ratio': ValueProxy(key='eval_full_to_train_steps_ratio', value=50),\n", + " 'experiment_dump_to_train_steps_ratio': ValueProxy(key='experiment_dump_to_train_steps_ratio', value=10),\n", + " 'is_training': ValueProxy(key='is_training', value=True),\n", + " 'log_every': ValueProxy(key='log_every', value=50),\n", + " 'max_steps': ValueProxy(key='max_steps', value=1500),\n", + " 'enable_h5_persistence': ValueProxy(key='enable_h5_persistence', value=True),\n", + " 'serving_grpc': ValueProxy(key='serving_grpc', value=True),\n", + " 'serving_cli': ValueProxy(key='serving_cli', value=True),\n", + " 'data': {'dataset_type': 'olivetti',\n", + " 'root_data_dir': '',\n", + " 'image_size': 64,\n", + " 'train_ratio': 0.8,\n", + " 'min_images_per_class': 2,\n", + " 'train_loader': {'batch_size': 32, 'shuffle': True, 'n_workers': 0},\n", + " 'test_loader': {'batch_size': 64, 'shuffle': False, 'n_workers': 0}},\n", + " 'model': {'backbone': 'resnet18',\n", + " 'pretrained': True,\n", + " 'freeze_backbone': True,\n", + " 'embedding_dim': 128,\n", + " 'head_hidden_dim': 256,\n", + " 'lr': 0.001,\n", + " 'weight_decay': 0.0001,\n", + " 'loss': 'triplet',\n", + " 'margin': 0.3},\n", + " 'root_log_dir': '/tmp/tmpu_lry8gs'}" + ] + }, + "metadata": {}, + "execution_count": 7 + } + ], + "source": [ + "# ============================================================\n", + "# Face Recognition - Training Config (inlined from config.yaml)\n", + "# ============================================================\n", + "config = {\n", + " \"experiment_name\": \"face_triplet_training\",\n", + " \"device\": \"auto\", # \"auto\" -> cuda if available, else cpu\n", + "\n", + " # Training schedule\n", + " \"eval_full_to_train_steps_ratio\": 50, # evaluation frequency (steps); -1 disables\n", + " \"experiment_dump_to_train_steps_ratio\": 10, # print running loss every N steps\n", + " \"is_training\": True, # start training immediately or not\n", + " \"log_every\": 50, # running-loss print cadence\n", + " \"max_steps\": 1500, # Colab cap so the training cell terminates;\n", + " # set to None to train open-ended (interrupt to stop)\n", + "\n", + " # WeightsLAB persistence\n", + " # \"root_log_dir\": leave unset -> auto temp dir\n", + " \"enable_h5_persistence\": True,\n", + "\n", + " # WeightsLAB serving\n", + " \"serving_grpc\": True,\n", + " \"serving_cli\": True,\n", + "\n", + " # --------------------------------------------------------\n", + " # Data\n", + " # --------------------------------------------------------\n", + " \"data\": {\n", + " # dataset_type options:\n", + " # \"olivetti\" - sklearn Olivetti Faces (40 ids, 400 imgs, 64x64 grey).\n", + " # Works offline, no download needed. Ideal for quick tests.\n", + " # \"lfw\" - LFW People via torchvision (download on first run).\n", + " # \"folder\" - Generic ImageFolder layout: root/{train,test}/{class}/*.jpg\n", + " \"dataset_type\": \"olivetti\",\n", + " \"root_data_dir\": \"\", # only required for lfw / folder\n", + " \"image_size\": 64, # images are resized to image_size x image_size\n", + " \"train_ratio\": 0.8, # olivetti only: fraction used for training\n", + " \"min_images_per_class\": 2, # identities with fewer samples are discarded\n", + "\n", + " \"train_loader\": {\n", + " \"batch_size\": 32,\n", + " \"shuffle\": True,\n", + " \"n_workers\": 0, # >0 for multi-process loading (not needed for Olivetti)\n", + " },\n", + " \"test_loader\": {\n", + " \"batch_size\": 64,\n", + " \"shuffle\": False,\n", + " \"n_workers\": 0,\n", + " },\n", + " },\n", + "\n", + " # --------------------------------------------------------\n", + " # Model\n", + " # --------------------------------------------------------\n", + " \"model\": {\n", + " # backbone: \"resnet18\" | \"resnet50\" | \"mobilenet_v3_small\"\n", + " \"backbone\": \"resnet18\",\n", + " \"pretrained\": True, # load ImageNet-pretrained weights\n", + " \"freeze_backbone\": True, # train the embedding head only (fast & effective)\n", + "\n", + " # Embedding head\n", + " \"embedding_dim\": 128, # output dim of the L2-normalised vector\n", + " \"head_hidden_dim\": 256, # hidden dim of the projection MLP\n", + "\n", + " # Optimiser (AdamW, trainable params only)\n", + " \"lr\": 0.001,\n", + " \"weight_decay\": 0.0001,\n", + "\n", + " # Triplet loss\n", + " \"loss\": \"triplet\",\n", + " \"margin\": 0.3, # triplet margin alpha\n", + " },\n", + "}\n", + "\n", + "wl.watch_or_edit(config, flag=\"hyperparameters\", defaults=config, poll_interval=1.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rHGGFVtgF399" + }, + "source": [ + "## 4. Datasets and tracked loaders\n", + "\n", + "Build the **Olivetti Faces** train/test splits and wrap the loaders with\n", + "`wl.watch_or_edit(..., flag=\"data\")` so every sample keeps its identity in WeightsLab.\n", + "Olivetti downloads automatically through scikit-learn (cached under `~/scikit_learn_data`)." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "91s0NMtEF399", + "outputId": "07e8d642-4be4-48bb-86cc-aebfd17171f2", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "downloading Olivetti faces from https://ndownloader.figshare.com/files/5976027 to /root/scikit_learn_data\n", + "16/07/2026-08:47:46.760 INFO:__main__:__init__: FaceDataset [olivetti / train] | samples=320 | classes=40\n", + "16/07/2026-08:47:46.867 INFO:__main__:__init__: FaceDataset [olivetti / test] | samples=80 | classes=40\n", + "\n", + "Dataset : olivetti | train=320 | test=80 | classes=40\n", + "16/07/2026-08:47:46.880 INFO:weightslab.data.data_samples_with_ops:__init__: [DataSampleTrackingWrapper] H5 persistence enabled at /tmp/tmpu_lry8gs/checkpoints/data/data.h5\n", + "16/07/2026-08:47:46.893 INFO:weightslab.data.data_samples_with_ops:__init__: Preloading sample statistics for PHYSICAL indices in split 'train_loader' with: preload_labels=True, preload_metadata=True...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "Initializing ledger for split 'train_loader': 0%| | 0/320 [00:00 the runtime device detected above).\n", + "if config.get(\"device\", \"auto\") == \"auto\":\n", + " config[\"device\"] = str(device)\n", + "\n", + "# Log dir (auto temp dir if unset).\n", + "if not config.get(\"root_log_dir\"):\n", + " config[\"root_log_dir\"] = tempfile.mkdtemp()\n", + " print(f\"No root_log_dir specified - using temp dir: {config['root_log_dir']}\")\n", + "os.makedirs(config[\"root_log_dir\"], exist_ok=True)\n", + "\n", + "eval_full_to_train_steps_ratio = int(config.get(\"eval_full_to_train_steps_ratio\", 50))\n", + "log_every = int(config.get(\"log_every\", 10))\n", + "enable_h5 = config.get(\"enable_h5_persistence\", True)\n", + "\n", + "# ---- Datasets ----\n", + "data_cfg = config.get(\"data\", {})\n", + "dataset_type = data_cfg.get(\"dataset_type\", \"olivetti\")\n", + "root_data = data_cfg.get(\"root_data_dir\", config.get(\"root_log_dir\", \".\"))\n", + "image_size = int(data_cfg.get(\"image_size\", 64))\n", + "min_imgs = int(data_cfg.get(\"min_images_per_class\", 2))\n", + "train_ratio = float(data_cfg.get(\"train_ratio\", 0.8))\n", + "\n", + "train_dataset = FaceDataset(\n", + " root=root_data,\n", + " dataset_type=dataset_type,\n", + " split=\"train\",\n", + " image_size=image_size,\n", + " train_ratio=train_ratio,\n", + " min_images_per_class=min_imgs,\n", + ")\n", + "test_dataset = FaceDataset(\n", + " root=root_data,\n", + " dataset_type=dataset_type,\n", + " split=\"test\",\n", + " image_size=image_size,\n", + " train_ratio=train_ratio,\n", + " min_images_per_class=min_imgs,\n", + ")\n", + "\n", + "print(\n", + " f\"\\nDataset : {dataset_type}\"\n", + " f\" | train={len(train_dataset)}\"\n", + " f\" | test={len(test_dataset)}\"\n", + " f\" | classes={train_dataset.num_classes}\"\n", + ")\n", + "\n", + "train_cfg = data_cfg.get(\"train_loader\", {})\n", + "test_cfg = data_cfg.get(\"test_loader\", {})\n", + "\n", + "# ---- WeightsLab-tracked data loaders ----\n", + "train_loader = wl.watch_or_edit(\n", + " train_dataset,\n", + " flag=\"data\",\n", + " loader_name=\"train_loader\",\n", + " batch_size=int(train_cfg.get(\"batch_size\", 32)),\n", + " shuffle=train_cfg.get(\"shuffle\", True),\n", + " is_training=True,\n", + " compute_hash=False,\n", + " num_workers=int(train_cfg.get(\"n_workers\", 0)),\n", + " enable_h5_persistence=enable_h5,\n", + ")\n", + "test_loader = wl.watch_or_edit(\n", + " test_dataset,\n", + " flag=\"data\",\n", + " loader_name=\"test_loader\",\n", + " batch_size=int(test_cfg.get(\"batch_size\", 64)),\n", + " shuffle=test_cfg.get(\"shuffle\", False),\n", + " is_training=False,\n", + " compute_hash=False,\n", + " num_workers=int(test_cfg.get(\"n_workers\", 0)),\n", + " enable_h5_persistence=enable_h5,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5JWo3BxFF399" + }, + "source": [ + "## 5. Model\n", + "\n", + "A pretrained ResNet-18 backbone (frozen) plus a small L2-normalised embedding head, trained\n", + "with online batch-hard **triplet loss**. Constructing it registers the graph and the loss\n", + "with WeightsLab." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "2J48fm6sF399", + "outputId": "2970e5b8-1f7c-4dc4-ffd1-f8c7f50c5444", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "100%|██████████| 44.7M/44.7M [00:00<00:00, 77.5MB/s]\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:47:49.494 INFO:__main__:_build_backbone: Backbone frozen ? training head only.\n", + "16/07/2026-08:47:49.505 INFO:weightslab.backend.model_interface:__init__: Using checkpoint manager from ledger\n", + "16/07/2026-08:47:49.509 INFO:__main__:__init__: FaceEmbeddingModel | backbone=resnet18 pretrained=True frozen=True | emb_dim=128 | trainable_params=164,736\n", + " Backbone : resnet18 (pretrained=True, frozen=True)\n", + " Emb dim : 128\n", + " Head dim : 256\n", + " Trainable : 164,736 params\n", + " Device : cpu\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/torch/nn/_reduction.py:51: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.\n", + " warnings.warn(warning.format(ret))\n" + ] + } + ], + "source": [ + "# ---- Model ----\n", + "model_cfg = config.get(\"model\", {})\n", + "model = FaceEmbeddingModel(\n", + " backbone_name=model_cfg.get(\"backbone\", \"resnet18\"),\n", + " embedding_dim=int(model_cfg.get(\"embedding_dim\", 128)),\n", + " head_hidden_dim=int(model_cfg.get(\"head_hidden_dim\", 256)),\n", + " lr=float(model_cfg.get(\"lr\", 1e-3)),\n", + " weight_decay=float(model_cfg.get(\"weight_decay\", 1e-4)),\n", + " freeze_backbone=model_cfg.get(\"freeze_backbone\", True),\n", + " pretrained=model_cfg.get(\"pretrained\", True),\n", + " margin=float(model_cfg.get(\"margin\", 0.3)),\n", + " device=config[\"device\"],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EmKUHOgJF399" + }, + "source": [ + "## 6. Train and evaluate steps\n", + "\n", + "`evaluate(...)` collects embeddings over a loader and computes verification / retrieval /\n", + "clustering signals; `train(...)` runs the batch-hard triplet loop. The\n", + "`guard_training_context` / `guard_testing_context` blocks tag each phase. The loop is capped\n", + "by `max_steps` so it terminates on Colab (pass `max_steps=None` to run open-ended)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "id": "1Cu9mwxEF399" + }, + "outputs": [], + "source": [ + "def evaluate(\n", + " model: FaceEmbeddingModel,\n", + " loader: torch.utils.data.DataLoader,\n", + " name: str = \"test\",\n", + ") -> Dict[str, float]:\n", + " \"\"\"Collect embeddings from loader and compute face recognition metrics.\"\"\"\n", + " print(f\"\\n{'=' * 55}\")\n", + " print(f\"Evaluation [{name}]\")\n", + " print(f\"{'=' * 55}\")\n", + "\n", + " all_embeddings: List[np.ndarray] = []\n", + " all_labels: List[np.ndarray] = []\n", + " all_uids: List[str] = []\n", + "\n", + " for images, uids, labels, _metadata in loader:\n", + " emb = model.get_embeddings(images) # (B, D)\n", + " all_embeddings.append(emb.numpy())\n", + " if isinstance(labels, torch.Tensor):\n", + " all_labels.append(labels.numpy())\n", + " else:\n", + " all_labels.append(np.array(labels))\n", + " all_uids.extend(uids if isinstance(uids, list) else list(uids))\n", + "\n", + " all_embeddings = np.concatenate(all_embeddings, axis=0)\n", + " all_labels = np.concatenate(all_labels, axis=0)\n", + "\n", + " metrics = FaceMetrics.compute_all_metrics(\n", + " ids=all_uids,\n", + " embeddings=all_embeddings,\n", + " labels=all_labels,\n", + " name=name,\n", + " )\n", + "\n", + " print(f\" verification_accuracy : {metrics.get('verification_accuracy', float('nan')):.4f}\")\n", + " print(f\" rank1_accuracy : {metrics.get('rank1_accuracy', float('nan')):.4f}\")\n", + " print(f\" FAR : {metrics.get('far', float('nan')):.4f}\")\n", + " print(f\" FRR : {metrics.get('frr', float('nan')):.4f}\")\n", + " print(f\" best_threshold : {metrics.get('best_threshold', float('nan')):.4f}\")\n", + " if \"num_clusters\" in metrics:\n", + " print(f\" num_clusters : {metrics['num_clusters']:.0f}\")\n", + " print(f\" noise_ratio : {metrics['noise_ratio']:.4f}\")\n", + " print(f\" mean_nn1_distance : {metrics['mean_nn1_distance']:.4f}\")\n", + "\n", + " return metrics\n", + "\n", + "\n", + "def train(\n", + " model: FaceEmbeddingModel,\n", + " train_loader: torch.utils.data.DataLoader,\n", + " test_loader: torch.utils.data.DataLoader,\n", + " eval_full_to_train_steps_ratio: int = 50,\n", + " log_every: int = 10,\n", + " loss_name: str = \"triplet\",\n", + " max_steps: int = None,\n", + ") -> Dict[str, Any]:\n", + " \"\"\"Training loop.\n", + "\n", + " Runs until `max_steps` optimizer steps have been taken (Colab-friendly), or\n", + " indefinitely when `max_steps` is None (interrupt the cell to stop). Periodic test\n", + " evaluation runs every eval_full_to_train_steps_ratio steps when test_loader is provided.\n", + " \"\"\"\n", + " print(\"\\n\" + \"=\" * 60)\n", + " print(\"Face Recognition Training\")\n", + " print(f\" Loss : {loss_name}\")\n", + " print(f\" Eval every : {eval_full_to_train_steps_ratio} steps\")\n", + " print(f\" Max steps : {'infinite (stop with Ctrl+C)' if max_steps is None else max_steps}\")\n", + " print(\"=\" * 60)\n", + "\n", + " data_iter = iter(train_loader)\n", + " losses: List[float] = []\n", + " eval_history: List[Dict[str, Any]] = []\n", + " step = 0\n", + "\n", + " try:\n", + " while max_steps is None or step < max_steps:\n", + " step += 1\n", + "\n", + " with guard_training_context:\n", + " # ---- Fetch next batch (cycle loader) ----\n", + " try:\n", + " images, batch_ids, labels, _metadata = next(data_iter)\n", + " except StopIteration:\n", + " data_iter = iter(train_loader)\n", + " images, batch_ids, labels, _metadata = next(data_iter)\n", + "\n", + " if not isinstance(batch_ids, list):\n", + " batch_ids = list(batch_ids)\n", + "\n", + " if isinstance(labels, torch.Tensor):\n", + " labels_tensor = labels.long()\n", + " else:\n", + " labels_tensor = torch.tensor(labels, dtype=torch.long)\n", + "\n", + " # ---- Gradient step ----\n", + " loss_val = model.train_step(\n", + " images=images,\n", + " labels=labels_tensor,\n", + " batch_ids=batch_ids,\n", + " loss_name=loss_name,\n", + " )\n", + " losses.append(loss_val)\n", + "\n", + " if step == 1 or step % max(1, log_every) == 0:\n", + " window = losses[-log_every:] if len(losses) >= log_every else losses\n", + " print(\n", + " f\"[train] step {step:>7d} \"\n", + " f\"| loss={loss_val:.6f} \"\n", + " f\"| running_mean={np.mean(window):.6f}\"\n", + " )\n", + "\n", + " should_eval = (\n", + " test_loader is not None\n", + " and (step % eval_full_to_train_steps_ratio == 0)\n", + " )\n", + "\n", + " if should_eval:\n", + " with guard_testing_context:\n", + " print(f\"\\n[eval@test] step {step}\")\n", + " metrics = evaluate(model=model, loader=test_loader, name=\"test\")\n", + " eval_history.append({\"step\": step, \"metrics\": metrics})\n", + "\n", + " except KeyboardInterrupt:\n", + " print(\"\\nInterrupted by user. Stopping training loop.\")\n", + "\n", + " summary = {\n", + " \"train_steps\": step,\n", + " \"train_loss_mean\": float(np.mean(losses)) if losses else float(\"nan\"),\n", + " \"train_loss_last\": float(losses[-1]) if losses else float(\"nan\"),\n", + " \"num_evals\": len(eval_history),\n", + " }\n", + "\n", + " print(\"\\nTraining summary:\")\n", + " for k, v in summary.items():\n", + " print(f\" {k}: {v}\")\n", + "\n", + " return summary" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wBOmcQBgF39-" + }, + "source": [ + "## 7. Serve and train\n", + "\n", + "`wl.serve(serving_grpc=True, serving_bore=True)` starts the background gRPC server and a\n", + "public `bore.pub` tunnel (the endpoint is printed below). `wl.start_training(...)` flips the\n", + "experiment into the training state, then the finite loop runs while streaming per-sample\n", + "signals. Leave this cell **running** while you watch it in the UI." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "N_88-lktF39-", + "outputId": "e736988d-2a95-4041-e9e5-414ec2861619", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:47:49.574 INFO:weightslab.trainer.trainer_services:_run_security_preflight: \n", + "\n", + "# #######################################\n", + "# #######################################\n", + "[gRPC] Security preflight checks:\n", + "\tTLS: DISABLED (unencrypted traffic)\n", + "\tAuth tokens: NONE configured\n", + "\t! WARNING: GRPC_TLS_ENABLED=0. Traffic will be unencrypted. Use only for development.\n", + "\t! WARNING: No GRPC_AUTH_TOKEN/GRPC_AUTH_TOKENS configured. Only transport-level trust (TLS/mTLS) will protect RPC access.\n", + "# #######################################\n", + "# #######################################\n", + "\n", + "16/07/2026-08:47:49.576 INFO:weightslab.trainer.trainer_services:grpc_serve: [gRPC] Watchdogs disabled via WEIGHTSLAB_DISABLE_WATCHDOGS.\n", + "16/07/2026-08:47:49.577 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Thread callback started\n", + "16/07/2026-08:47:49.580 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Creating ThreadPoolExecutor with 6 worker threads (n_workers_grpc=None, max_concurrent_rpcs=None)\n", + "16/07/2026-08:47:49.578 INFO:weightslab.trainer.trainer_services:grpc_serve: [gRPC] Server started with watchdogs disabled (host=0.0.0.0 port=50051 workers=None)\n", + "16/07/2026-08:47:49.585 INFO:weightslab.tunnel:_ensure_bore: Downloading bore v0.6.0 (bore-v0.6.0-x86_64-unknown-linux-musl.tar.gz)...\n", + "16/07/2026-08:47:49.729 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Server object created\n", + "16/07/2026-08:47:49.788 INFO:weightslab.trainer.services.agent.agent:__init__: Initializing DataManipulationAgent\n", + "16/07/2026-08:47:49.841 WARNING:weightslab.trainer.services.agent.agent:_load_config: Error loading config from /usr/local/lib/python3.12/dist-packages: [Errno 21] Is a directory: '/usr/local/lib/python3.12/dist-packages'\n", + "16/07/2026-08:47:49.853 INFO:weightslab.trainer.services.agent.agent:_load_config: \n", + "\n", + "# #######################################\n", + "# #######################################\n", + "Agent initialized from configuration /content/agent_config.yaml: \n", + "\tFinal Agent Configuration: Preferred Provider=openrouter, \n", + "\tFallback to Local=True, \n", + "\tOpenRouter Model=~google/gemini-flash-latest with:\n", + "\t\tAPI Key=None\n", + "\t\tBase URL=https://openrouter.ai/api/v1, \n", + "\tOllama Model=llama3.2:3b\n", + "# #######################################\n", + "# #######################################\n", + "\n", + "16/07/2026-08:47:49.860 INFO:weightslab.trainer.services.agent.agent:_setup_providers: Setting up Ollama with model llama3.2:3b\n", + "16/07/2026-08:47:50.171 INFO:weightslab.trainer.services.agent.agent:_setup_providers: [Agent] Ollama enabled: llama3.2:3b\n", + "16/07/2026-08:47:50.173 INFO:weightslab.trainer.services.data_service:_build_preview_cache: [PreviewCache] Building 64×64 or less or less preview cache for 400 samples …\n", + "16/07/2026-08:47:50.173 INFO:weightslab.trainer.services.data_service:__init__: DataService initialized.\n", + "16/07/2026-08:47:50.175 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Servicer added\n", + "16/07/2026-08:47:50.176 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Attempting to bind to 0.0.0.0:50051\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\r[PreviewCache]: 0%| | 0/400 [00:00` printed in the Expose section:\n", + "\n", + "```bash\n", + "pip install weightslab\n", + "weightslab start # Terminal 1 - opens http://localhost:5173\n", + "weightslab tunnel bore.pub:12345 # Terminal 2 - the host:port from the Expose section\n", + "```\n", + "\n", + "Then open **http://localhost:5173**. Prefer local-only? Run this example directly on your machine\n", + "(`weightslab start example` selects a bundled example) and launch the UI next to it - no tunnel." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EW46PivEF39-" + }, + "source": [ + "---\n", + "\n", + "
\n", + "Crafted by GrayboxTech - if WeightsLab helps you catch a bad label, drop us a star on GitHub.\n", + "
" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "wl-clustering.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/PyTorch/wl-detection.ipynb b/weightslab/examples/Notebooks/PyTorch/wl-detection.ipynb new file mode 100644 index 00000000..d09d7550 --- /dev/null +++ b/weightslab/examples/Notebooks/PyTorch/wl-detection.ipynb @@ -0,0 +1,4691 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Z2kgyn-s_xBa" + }, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Welcome to the WeightsLab Object Detection (Penn-Fudan) notebook! WeightsLab traces training signals back to the exact samples producing them. Browse the Docs for details.
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C_WnDPZv_xBc" + }, + "source": [ + "# Object Detection (Penn-Fudan) with WeightsLab\n", + "\n", + "This notebook trains a small detector (MobileNetV3-Small backbone + a lightweight detection head) on the **Penn-Fudan** pedestrian dataset and instruments it with WeightsLab, so every training signal is traced **back to the exact samples** producing it.\n", + "\n", + "WeightsLab records **per-sample** detection loss and **per-instance** IoU (one value per ground-truth box) live, so each predicted box is traceable in Studio - you can rank the worst samples, spot bad boxes, and curate the dataset **without restarting training**.\n", + "\n", + "> Data: the Penn-Fudan dataset (~170 images, one class: *person*) is **downloaded automatically on first run** - nothing to upload.\n", + "\n", + "### What you'll do\n", + "1. Install WeightsLab.\n", + "2. Inline the dataset, detector, and loss/IoU criterions (previously the example's `utils/` package).\n", + "3. Set every knob in one **config** dict (like a `config.yaml`).\n", + "4. Wrap the model, optimizer, dataloaders, loss, and IoU metrics with the SDK.\n", + "5. Train while per-sample and per-instance signals are captured, then open the live **Weights Studio** UI." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dpQhH7Jd_xBc" + }, + "source": [ + "## Setup\n", + "\n", + "On Colab, pick a **T4 GPU** runtime (`Runtime -> Change runtime type -> T4 GPU`)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3nlJ2-Oj_xBc" + }, + "source": [ + "\"PyPI\n", + "\"PyPI\n", + "\"PyPI" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "WptCnGqv_xBd", + "outputId": "617e6869-1797-49dc-96b9-525dfe8e7b35", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Looking in indexes: https://test.pypi.org/simple/, https://pypi.org/simple/\n", + "Requirement already satisfied: weightslab==1.3.3.dev7 in /usr/local/lib/python3.12/dist-packages (1.3.3.dev7)\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (3.10.2)\n", + "Requirement already satisfied: torch<=2.9,>=2.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (2.9.0)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (0.24.0)\n", + "Requirement already satisfied: torchmetrics>=1.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.9.0)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (0.21)\n", + "Requirement already satisfied: onnx<=1.20,>=1.15 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.20.0)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.4.9)\n", + "Requirement already satisfied: langchain-ollama<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.1.0)\n", + "Requirement already satisfied: langchain-openai<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev7) (1.3.5)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev7) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.16.2)\n", + "Requirement already satisfied: ollama<1.0.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev7) (0.6.2)\n", + "Requirement already satisfied: openai<3.0.0,>=2.45.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev7) (2.45.0)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev7) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev7) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev7) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev7) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev7) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev7) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev7) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev7) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev7) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev7) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev7) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.5.0)\n", + "Requirement already satisfied: lightning-utilities>=0.15.3 in /usr/local/lib/python3.12/dist-packages (from torchmetrics>=1.9->weightslab==1.3.3.dev7) (0.15.3)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev7) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev7) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev7) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev7) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev7) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev7) (2.0.0)\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install -q weightslab\n", + "%pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev7\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KtEZwMxI_xBd" + }, + "source": [ + "## 1. Imports\n", + "\n", + "`weightslab` is imported as `wl`. The two `guard_*_context` managers scope a block as training vs. evaluation so signals are attributed to the right phase. Every external dependency used by the inlined modules is imported here." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "NHwcL67i_xBd", + "outputId": "336670cc-0cab-4abf-b13f-91f6babe5c14", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:32:40.383 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmp_zk4n7mc/weightslab_logs/weightslab_20260716_083240.log\n", + "16/07/2026-08:32:40.387 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-08:32:40.388 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev7 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-08:32:40.394 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n", + "Using device: cpu\n" + ] + } + ], + "source": [ + "import os\n", + "import ssl\n", + "import zipfile\n", + "import logging\n", + "import tempfile\n", + "import urllib.request\n", + "\n", + "import numpy as np\n", + "import tqdm\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from torch import optim\n", + "from torch.utils.data import Dataset\n", + "from torchvision import transforms\n", + "from torchvision.models import mobilenet_v3_small, MobileNet_V3_Small_Weights\n", + "from PIL import Image\n", + "\n", + "import weightslab as wl\n", + "from weightslab.components.global_monitoring import (\n", + " guard_training_context,\n", + " guard_testing_context,\n", + ")\n", + "\n", + "logging.basicConfig(level=logging.ERROR)\n", + "logging.getLogger(\"PIL\").setLevel(logging.INFO)\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "print(f\"Using device: {device}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OOYtb2D2_xBe" + }, + "source": [ + "## 2. Inlined helper modules (from the example's `utils/`)\n", + "\n", + "The example ships a small `utils/` package; its three modules are inlined below so the notebook is fully self-contained. Only the intra-package imports are dropped - those names (`decode_grid`, `det_collate`, the dataset and the criterions) are now notebook-global.\n", + "\n", + "- **`utils/data.py`** - the Penn-Fudan detection dataset (auto-downloads on first use) + the `det_collate` collate fn.\n", + "- **`utils/model.py`** - a MobileNetV3-Small backbone + a small grid detection head.\n", + "- **`utils/criterions.py`** - per-sample YOLO-style detection loss and per-sample / per-instance IoU." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "csaORA30_xBe" + }, + "outputs": [], + "source": [ + "# ===== utils/data.py =====\n", + "import os\n", + "import ssl\n", + "import zipfile\n", + "import urllib.request\n", + "\n", + "import numpy as np\n", + "import torch\n", + "\n", + "from torchvision import transforms\n", + "\n", + "from torch.utils.data import Dataset\n", + "\n", + "from PIL import Image\n", + "\n", + "\n", + "# =============================================================================\n", + "# Penn-Fudan Pedestrian detection dataset\n", + "# =============================================================================\n", + "# A small, real object-detection dataset (~170 photos, one class: \"person\").\n", + "# It ships per-instance segmentation masks; we derive an axis-aligned bounding\n", + "# box per pedestrian from each mask. Downloaded + extracted on first use.\n", + "#\n", + "# On-disk layout after extraction:\n", + "# /PennFudanPed/\n", + "# PNGImages/FudanPed00001.png ...\n", + "# PedMasks/FudanPed00001_mask.png ... # pixel value k = k-th pedestrian, 0 = bg\n", + "#\n", + "# WL renders detection targets/predictions from a per-sample [N, 6] array\n", + "# ``[x1, y1, x2, y2, class_id, confidence]`` normalized to [0, 1] (GT conf = 1.0)\n", + "# — see ``get_items`` below and ``data_service.py`` (task_type == \"detection\").\n", + "\n", + "CLASS_NAMES = [\"person\"]\n", + "\n", + "_URL = \"https://www.cis.upenn.edu/~jshi/ped_html/PennFudanPed.zip\"\n", + "\n", + "# ImageNet statistics — the MobileNetV3 backbone is pretrained with these, so we\n", + "# normalize model inputs the same way. (The UI still shows the original PNG.)\n", + "IMAGENET_MEAN = [0.485, 0.456, 0.406]\n", + "IMAGENET_STD = [0.229, 0.224, 0.225]\n", + "\n", + "\n", + "def download_penn_fudan(root):\n", + " \"\"\"Download + extract Penn-Fudan into /PennFudanPed (idempotent).\"\"\"\n", + " base = os.path.join(root, \"PennFudanPed\")\n", + " if os.path.isdir(os.path.join(base, \"PNGImages\")):\n", + " return base\n", + "\n", + " os.makedirs(root, exist_ok=True)\n", + " zip_path = os.path.join(root, \"PennFudanPed.zip\")\n", + "\n", + " if not os.path.exists(zip_path):\n", + " print(f\"[data] Downloading Penn-Fudan dataset to {zip_path} ...\", flush=True)\n", + " try:\n", + " urllib.request.urlretrieve(_URL, zip_path)\n", + " except Exception as e:\n", + " # Some corporate environments break TLS verification - retry unverified.\n", + " print(f\"[data] TLS verification failed ({e}); retrying without verification.\", flush=True)\n", + " ctx = ssl._create_unverified_context()\n", + " with urllib.request.urlopen(_URL, context=ctx) as resp, open(zip_path, \"wb\") as fh:\n", + " fh.write(resp.read())\n", + "\n", + " print(\"[data] Extracting Penn-Fudan ...\", flush=True)\n", + " with zipfile.ZipFile(zip_path) as zf:\n", + " zf.extractall(root)\n", + " return base\n", + "\n", + "\n", + "def _boxes_from_mask(mask_path):\n", + " \"\"\"Derive one bbox per pedestrian from a Penn-Fudan instance mask.\n", + "\n", + " Returns (boxes_px [N, 4] int xyxy, height, width). Background (0) skipped.\n", + " \"\"\"\n", + " mask = np.array(Image.open(mask_path))\n", + " h, w = mask.shape[:2]\n", + " obj_ids = np.unique(mask)\n", + " obj_ids = obj_ids[obj_ids != 0] # drop background\n", + "\n", + " boxes = []\n", + " for oid in obj_ids:\n", + " ys, xs = np.where(mask == oid)\n", + " if xs.size == 0:\n", + " continue\n", + " x1, x2 = int(xs.min()), int(xs.max())\n", + " y1, y2 = int(ys.min()), int(ys.max())\n", + " if x2 > x1 and y2 > y1:\n", + " boxes.append([x1, y1, x2, y2])\n", + " return np.asarray(boxes, dtype=np.float32).reshape(-1, 4), h, w\n", + "\n", + "\n", + "class PennFudanDetectionDataset(Dataset):\n", + " \"\"\"Pedestrian bounding-box detection over the Penn-Fudan images.\n", + "\n", + " Args:\n", + " root: directory to download/extract the dataset into.\n", + " split: \"train\" or \"val\" (deterministic split of the 170 images).\n", + " image_size: square resize fed to the model.\n", + " val_fraction: fraction of images held out for validation.\n", + " max_samples: optional cap on the split size (for quick runs).\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " root,\n", + " split=\"train\",\n", + " num_classes=1,\n", + " image_size=256,\n", + " val_fraction=0.2,\n", + " max_samples=None,\n", + " ):\n", + " super().__init__()\n", + " self.root = root\n", + " self.split = split\n", + " self.num_classes = num_classes\n", + " self.image_size = image_size\n", + " # Explicit task type; bypasses WL's label-shape heuristic so bboxes are\n", + " # rendered as detection overlays (not mistaken for classification).\n", + " self.task_type = \"detection\"\n", + " self.class_names = CLASS_NAMES[:num_classes]\n", + "\n", + " base = download_penn_fudan(root)\n", + " img_dir = os.path.join(base, \"PNGImages\")\n", + " mask_dir = os.path.join(base, \"PedMasks\")\n", + "\n", + " all_imgs = sorted(f for f in os.listdir(img_dir) if f.lower().endswith(\".png\"))\n", + "\n", + " # Deterministic train/val split: every k-th image goes to val.\n", + " k = max(2, int(round(1.0 / max(val_fraction, 1e-6))))\n", + " if split == \"val\":\n", + " selected = all_imgs[::k]\n", + " else:\n", + " val_set = set(all_imgs[::k])\n", + " selected = [f for f in all_imgs if f not in val_set]\n", + "\n", + " selected = selected[:max_samples] if max_samples != None else selected\n", + "\n", + " self.images = []\n", + " self.masks = []\n", + " for fname in selected:\n", + " base_name, _ = os.path.splitext(fname)\n", + " mask_path = os.path.join(mask_dir, base_name + \"_mask.png\")\n", + " if os.path.exists(mask_path):\n", + " self.images.append(os.path.join(img_dir, fname))\n", + " self.masks.append(mask_path)\n", + "\n", + " if len(self.images) == 0:\n", + " raise RuntimeError(f\"No image/mask pairs found under {base}\")\n", + "\n", + " self.image_transform = transforms.Compose(\n", + " [\n", + " transforms.Resize((image_size, image_size), interpolation=Image.BILINEAR),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),\n", + " ]\n", + " )\n", + "\n", + " def __len__(self):\n", + " return len(self.images)\n", + "\n", + " def _load_boxes(self, mask_path):\n", + " \"\"\"Read mask → [N, 6] float32 = [x1, y1, x2, y2, cls=0, conf=1.0] (normalized).\"\"\"\n", + " boxes_px, h, w = _boxes_from_mask(mask_path)\n", + " if boxes_px.shape[0] == 0:\n", + " return np.zeros((0, 6), dtype=np.float32)\n", + " norm = boxes_px.copy()\n", + " norm[:, [0, 2]] /= float(w)\n", + " norm[:, [1, 3]] /= float(h)\n", + " n = norm.shape[0]\n", + " cls = np.zeros((n, 1), dtype=np.float32) # single class: person\n", + " conf = np.ones((n, 1), dtype=np.float32)\n", + " return np.concatenate([norm, cls, conf], axis=1).astype(np.float32)\n", + "\n", + " def __getitem__(self, idx):\n", + " \"\"\"Returns (item, uid, target, metadata).\n", + "\n", + " - item: normalized image tensor [C, H, W]\n", + " - uid: unique sample id (string)\n", + " - target: [N, 6] float32 = [x1, y1, x2, y2, class_id, confidence]\n", + " - metadata: dict with source paths\n", + " \"\"\"\n", + " return self.get_items(idx, include_metadata=True, include_labels=True, include_images=True)\n", + "\n", + " def get_items(self, idx, include_metadata=False, include_labels=False, include_images=False):\n", + " img_path = self.images[idx]\n", + " mask_path = self.masks[idx]\n", + " uid = os.path.splitext(os.path.basename(img_path))[0]\n", + "\n", + " metadata = {\n", + " \"img_path\": img_path,\n", + " \"mask_path\": mask_path,\n", + " } if include_metadata else None\n", + "\n", + " img_t = None\n", + " if include_images:\n", + " img = Image.open(img_path).convert(\"RGB\")\n", + " img_t = self.image_transform(img)\n", + "\n", + " target = None\n", + " if include_labels:\n", + " target = self._load_boxes(mask_path)\n", + "\n", + " return img_t, uid, target, metadata\n", + "\n", + "\n", + "def det_collate(batch):\n", + " \"\"\"Collate WL per-sample tuples for object detection.\n", + "\n", + " A sample owns a variable number of boxes, so targets cannot be stacked. We\n", + " keep them as a Python list (one [N_i, 6] tensor per sample), exactly the\n", + " layout WL's per-instance helpers expect (``targets[s]`` iterates that\n", + " sample's boxes in annotation order).\n", + "\n", + " Returns:\n", + " images: FloatTensor [B, C, H, W]\n", + " ids: list[str] of length B\n", + " targets: list[B] of [N_i, 6] float tensors ([x1, y1, x2, y2, cls, conf])\n", + " metas: list[B] of metadata dicts\n", + " \"\"\"\n", + " images = torch.stack([b[0] for b in batch], dim=0)\n", + " ids = [b[1] for b in batch]\n", + " targets = [\n", + " torch.as_tensor(b[2], dtype=torch.float32)\n", + " if not isinstance(b[2], torch.Tensor) else b[2].float()\n", + " for b in batch\n", + " ]\n", + " metas = [b[3] if len(b) > 3 else None for b in batch]\n", + " return images, ids, targets, metas" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "GFZWMzNi_xBf" + }, + "outputs": [], + "source": [ + "# ===== utils/model.py =====\n", + "# =============================================================================\n", + "# Small single-shot grid detector on a pretrained MobileNetV3-Small backbone\n", + "# =============================================================================\n", + "# A frozen/fine-tuned ImageNet-pretrained backbone extracts features; a tiny\n", + "# detection head turns them into an S x S grid where each cell predicts ONE box:\n", + "# (objectness, tx, ty, tw, th, class_logits...).\n", + "#\n", + "# Encoding (all coordinates normalized to the [0, 1] image frame):\n", + "# * objectness = sigmoid(t_obj) -> P(box present in this cell)\n", + "# * cx = (col + sigmoid(tx)) / S -> box center, x\n", + "# * cy = (row + sigmoid(ty)) / S -> box center, y\n", + "# * w = sigmoid(tw) -> box width (fraction of image)\n", + "# * h = sigmoid(th) -> box height (fraction of image)\n", + "# * class = softmax(class_logits)\n", + "#\n", + "# Raw forward output keeps logits (loss applies the activations); `decode`\n", + "# turns logits into xyxy boxes for metrics and UI rendering.\n", + "import torch\n", + "import torch.nn as nn\n", + "\n", + "from torchvision.models import mobilenet_v3_small, MobileNet_V3_Small_Weights\n", + "\n", + "\n", + "def decode_grid(outputs, grid_size):\n", + " \"\"\"Decode raw grid logits -> per-cell xyxy boxes, objectness and class probs.\n", + "\n", + " Shared by the model (``SmallDetector.decode``) and the criterions so the\n", + " encoding lives in exactly one place.\n", + "\n", + " Args:\n", + " outputs: [B, S, S, 5 + num_classes] raw logits.\n", + " grid_size: S.\n", + "\n", + " Returns:\n", + " boxes: [B, S, S, 4] xyxy in [0, 1]\n", + " obj: [B, S, S] objectness probability\n", + " cls_probs: [B, S, S, num_classes] class probabilities\n", + " \"\"\"\n", + " B, S, _, _ = outputs.shape\n", + " device = outputs.device\n", + "\n", + " obj = torch.sigmoid(outputs[..., 0])\n", + " tx = torch.sigmoid(outputs[..., 1])\n", + " ty = torch.sigmoid(outputs[..., 2])\n", + " w = torch.sigmoid(outputs[..., 3])\n", + " h = torch.sigmoid(outputs[..., 4])\n", + " cls_probs = torch.softmax(outputs[..., 5:], dim=-1)\n", + "\n", + " # Cell-origin grid (col=x, row=y).\n", + " cols = torch.arange(S, device=device).view(1, 1, S).expand(B, S, S)\n", + " rows = torch.arange(S, device=device).view(1, S, 1).expand(B, S, S)\n", + "\n", + " cx = (cols + tx) / S\n", + " cy = (rows + ty) / S\n", + " x1 = (cx - w / 2).clamp(0, 1)\n", + " y1 = (cy - h / 2).clamp(0, 1)\n", + " x2 = (cx + w / 2).clamp(0, 1)\n", + " y2 = (cy + h / 2).clamp(0, 1)\n", + "\n", + " boxes = torch.stack([x1, y1, x2, y2], dim=-1)\n", + " return boxes, obj, cls_probs\n", + "\n", + "\n", + "class SmallDetector(nn.Module):\n", + " def __init__(\n", + " self,\n", + " in_channels=3,\n", + " num_classes=1,\n", + " image_size=256,\n", + " grid_size=8,\n", + " pretrained=True,\n", + " freeze_backbone=True,\n", + " ):\n", + " super().__init__()\n", + " # For WeightsLab\n", + " self.task_type = \"detection\"\n", + " self.num_classes = num_classes\n", + " self.class_names = [\"person\"][:num_classes]\n", + " self.grid_size = grid_size\n", + " self.image_size = image_size\n", + " self.input_shape = (1, in_channels, image_size, image_size)\n", + "\n", + " # Channels per cell: objectness(1) + box(4) + class logits(num_classes)\n", + " self.preds_per_cell = 5 + num_classes\n", + "\n", + " # --- Pretrained backbone (ImageNet) ---\n", + " weights = MobileNet_V3_Small_Weights.IMAGENET1K_V1 if pretrained else None\n", + " backbone = mobilenet_v3_small(weights=weights)\n", + " self.backbone = backbone.features # [B, 576, H/32, W/32]\n", + " backbone_out_ch = 576\n", + "\n", + " self.freeze_backbone = freeze_backbone\n", + " if freeze_backbone:\n", + " for p in self.backbone.parameters():\n", + " p.requires_grad = False\n", + "\n", + " # --- Detection head (lightweight, trained from scratch) ---\n", + " self.neck = nn.Sequential(\n", + " nn.Conv2d(backbone_out_ch, 256, 3, padding=1, bias=False),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(inplace=True),\n", + " nn.Conv2d(256, 128, 3, padding=1, bias=False),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(inplace=True),\n", + " )\n", + " self.head = nn.Conv2d(128, self.preds_per_cell, 1)\n", + "\n", + " def train(self, mode=True):\n", + " \"\"\"Keep a frozen backbone in eval mode (so its BatchNorm stats stay fixed).\"\"\"\n", + " super().train(mode)\n", + " if self.freeze_backbone:\n", + " self.backbone.eval()\n", + " return self\n", + "\n", + " def forward(self, x):\n", + " \"\"\"Returns raw logits [B, S, S, 5 + num_classes].\"\"\"\n", + " if self.freeze_backbone:\n", + " with torch.no_grad():\n", + " feat = self.backbone(x)\n", + " else:\n", + " feat = self.backbone(x)\n", + "\n", + " feat = self.neck(feat)\n", + " out = self.head(feat) # [B, preds_per_cell, S', S']\n", + "\n", + " # Resize feature grid to the configured grid_size.\n", + " if out.shape[-1] != self.grid_size or out.shape[-2] != self.grid_size:\n", + " out = nn.functional.adaptive_avg_pool2d(out, self.grid_size)\n", + " # [B, C, S, S] -> [B, S, S, C]\n", + " return out.permute(0, 2, 3, 1).contiguous()\n", + "\n", + " def decode(self, outputs):\n", + " \"\"\"Decode raw logits -> per-cell xyxy boxes, objectness, class probs.\"\"\"\n", + " return decode_grid(outputs, self.grid_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "GQpthcwy_xBf" + }, + "outputs": [], + "source": [ + "# ===== utils/criterions.py =====\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "\n", + "\n", + "# =============================================================================\n", + "# Per-instance / per-sample detection criterions (YOLO-v1 style loss + IoU)\n", + "# =============================================================================\n", + "# The detection dataset yields, per sample, a [N, 6] target tensor\n", + "# ``[x1, y1, x2, y2, class_id, confidence]`` (normalized to [0, 1]). Each GT box\n", + "# is assigned to the grid cell containing its center; that cell is \"responsible\"\n", + "# for predicting the box.\n", + "#\n", + "# * PerSampleDetectionLoss -> one differentiable loss scalar per sample ([B]),\n", + "# wrapped with ``per_sample=True`` (the value WL backprops + dashboards).\n", + "# * PerSampleIoU -> mean IoU over a sample's boxes ([B]), a metric.\n", + "# * PerInstanceIoU -> flat tensor of one IoU per GT box (sample-major\n", + "# order), wrapped with ``per_instance=True`` so WL auto-saves it at\n", + "# (sample_id, annotation_id). The ordering matches the per-sample target\n", + "# iteration, so the wrapper's auto ``batch_idx`` maps each value correctly.\n", + "\n", + "_EPS = 1e-6\n", + "_LAMBDA_COORD = 5.0\n", + "_LAMBDA_NOOBJ = 0.5\n", + "\n", + "\n", + "def box_iou_xyxy(a, b):\n", + " \"\"\"IoU between two aligned sets of xyxy boxes. a, b: [..., 4] -> [...].\"\"\"\n", + " x1 = torch.maximum(a[..., 0], b[..., 0])\n", + " y1 = torch.maximum(a[..., 1], b[..., 1])\n", + " x2 = torch.minimum(a[..., 2], b[..., 2])\n", + " y2 = torch.minimum(a[..., 3], b[..., 3])\n", + "\n", + " inter = (x2 - x1).clamp(min=0) * (y2 - y1).clamp(min=0)\n", + " area_a = (a[..., 2] - a[..., 0]).clamp(min=0) * (a[..., 3] - a[..., 1]).clamp(min=0)\n", + " area_b = (b[..., 2] - b[..., 0]).clamp(min=0) * (b[..., 3] - b[..., 1]).clamp(min=0)\n", + " union = area_a + area_b - inter + _EPS\n", + " return inter / union\n", + "\n", + "\n", + "def _responsible_cells(boxes, S):\n", + " \"\"\"Map GT boxes -> their responsible (row, col) cell and center offsets.\n", + "\n", + " Args:\n", + " boxes: [N, 4] xyxy in [0, 1].\n", + " S: grid size.\n", + "\n", + " Returns:\n", + " rows, cols: [N] long, the responsible cell indices.\n", + " off_x, off_y: [N] center offset within the cell, in [0, 1).\n", + " w, h: [N] box size as a fraction of the image.\n", + " \"\"\"\n", + " cx = (boxes[:, 0] + boxes[:, 2]) / 2\n", + " cy = (boxes[:, 1] + boxes[:, 3]) / 2\n", + " w = (boxes[:, 2] - boxes[:, 0]).clamp(_EPS, 1.0)\n", + " h = (boxes[:, 3] - boxes[:, 1]).clamp(_EPS, 1.0)\n", + "\n", + " cols = (cx * S).long().clamp(0, S - 1)\n", + " rows = (cy * S).long().clamp(0, S - 1)\n", + " off_x = (cx * S - cols).clamp(0, 1)\n", + " off_y = (cy * S - rows).clamp(0, 1)\n", + " return rows, cols, off_x, off_y, w, h\n", + "\n", + "\n", + "def _per_sample_loss(outputs, targets, num_classes, weights=None):\n", + " \"\"\"YOLO-v1 style loss, returned as one scalar per sample ([B], with grad).\"\"\"\n", + " B, S = outputs.shape[0], outputs.shape[1]\n", + " device = outputs.device\n", + "\n", + " obj_logit = outputs[..., 0] # [B, S, S]\n", + " tx = torch.sigmoid(outputs[..., 1])\n", + " ty = torch.sigmoid(outputs[..., 2])\n", + " w_pred = torch.sigmoid(outputs[..., 3])\n", + " h_pred = torch.sigmoid(outputs[..., 4])\n", + " cls_logits = outputs[..., 5:] # [B, S, S, C]\n", + "\n", + " if weights is not None:\n", + " weights = torch.as_tensor(weights, device=device, dtype=outputs.dtype)\n", + "\n", + " losses = []\n", + " for s in range(B):\n", + " tgt = targets[s]\n", + " tgt = torch.as_tensor(tgt, device=device, dtype=outputs.dtype)\n", + " if tgt.ndim == 1:\n", + " tgt = tgt.view(-1, 6) if tgt.numel() else tgt.view(0, 6)\n", + "\n", + " obj_target = torch.zeros((S, S), device=device, dtype=outputs.dtype)\n", + "\n", + " coord_loss = torch.zeros((), device=device)\n", + " class_loss = torch.zeros((), device=device)\n", + "\n", + " if tgt.numel() > 0:\n", + " boxes = tgt[:, :4]\n", + " cls_ids = tgt[:, 4].long().clamp(0, num_classes - 1)\n", + " rows, cols, off_x, off_y, gw, gh = _responsible_cells(boxes, S)\n", + "\n", + " obj_target[rows, cols] = 1.0\n", + "\n", + " # Localization: center offset (linear) + size in sqrt space (YOLO trick\n", + " # so small-box errors weigh as much as large-box ones).\n", + " coord = (\n", + " (tx[s, rows, cols] - off_x) ** 2\n", + " + (ty[s, rows, cols] - off_y) ** 2\n", + " + (torch.sqrt(w_pred[s, rows, cols] + _EPS) - torch.sqrt(gw + _EPS)) ** 2\n", + " + (torch.sqrt(h_pred[s, rows, cols] + _EPS) - torch.sqrt(gh + _EPS)) ** 2\n", + " )\n", + " coord_loss = _LAMBDA_COORD * coord.sum()\n", + "\n", + " ce = F.cross_entropy(\n", + " cls_logits[s, rows, cols], cls_ids, reduction=\"none\"\n", + " )\n", + " if weights is not None:\n", + " ce = ce * weights[cls_ids]\n", + " class_loss = ce.sum()\n", + "\n", + " # Objectness BCE over the whole grid; down-weight the (many) empty cells.\n", + " obj_weight = torch.where(\n", + " obj_target > 0,\n", + " torch.ones_like(obj_target),\n", + " torch.full_like(obj_target, _LAMBDA_NOOBJ),\n", + " )\n", + " obj_loss = (\n", + " F.binary_cross_entropy_with_logits(\n", + " obj_logit[s], obj_target, reduction=\"none\"\n", + " ) * obj_weight\n", + " ).sum()\n", + "\n", + " losses.append(coord_loss + class_loss + obj_loss)\n", + "\n", + " return torch.stack(losses)\n", + "\n", + "\n", + "def _per_box_iou(outputs, targets, grid_size):\n", + " \"\"\"IoU of each GT box against its responsible cell's decoded prediction.\n", + "\n", + " Returns a list[B] of 1-D tensors (one IoU per box for that sample, in\n", + " annotation order). Detached — this is a metric, not a loss.\n", + " \"\"\"\n", + " boxes_grid, _, _ = decode_grid(outputs, grid_size) # [B, S, S, 4]\n", + " B = outputs.shape[0]\n", + " S = grid_size\n", + " device = outputs.device\n", + "\n", + " per_sample = []\n", + " for s in range(B):\n", + " tgt = torch.as_tensor(targets[s], device=device, dtype=outputs.dtype)\n", + " if tgt.numel() == 0:\n", + " per_sample.append(torch.zeros(0, device=device))\n", + " continue\n", + " if tgt.ndim == 1:\n", + " tgt = tgt.view(-1, 6)\n", + "\n", + " gt_boxes = tgt[:, :4]\n", + " rows, cols, _, _, _, _ = _responsible_cells(gt_boxes, S)\n", + " pred_boxes = boxes_grid[s, rows, cols] # [N, 4]\n", + " ious = box_iou_xyxy(pred_boxes, gt_boxes) # [N]\n", + " per_sample.append(ious.detach())\n", + "\n", + " return per_sample\n", + "\n", + "\n", + "class PerSampleDetectionLoss(nn.Module):\n", + " \"\"\"Total detection loss aggregated to one value per sample ([B]).\"\"\"\n", + "\n", + " def __init__(self, num_classes, grid_size, weights=None):\n", + " super().__init__()\n", + " self.num_classes = num_classes\n", + " self.grid_size = grid_size\n", + " self.register_buffer(\n", + " \"weights\", torch.tensor(weights) if weights is not None else None\n", + " )\n", + "\n", + " def forward(self, outputs, targets):\n", + " return _per_sample_loss(outputs, targets, self.num_classes, weights=self.weights)\n", + "\n", + "\n", + "class PerSampleIoU(nn.Module):\n", + " \"\"\"Mean IoU over a sample's boxes -> one value per sample ([B]).\"\"\"\n", + "\n", + " def __init__(self, num_classes, grid_size):\n", + " super().__init__()\n", + " self.num_classes = num_classes\n", + " self.grid_size = grid_size\n", + "\n", + " def forward(self, outputs, targets):\n", + " per_sample = _per_box_iou(outputs, targets, self.grid_size)\n", + " out = [\n", + " v.mean() if v.numel() > 0 else torch.zeros((), device=outputs.device)\n", + " for v in per_sample\n", + " ]\n", + " return torch.stack(out).detach()\n", + "\n", + "\n", + "class PerInstanceIoU(nn.Module):\n", + " \"\"\"IoU per GT box -> flat tensor [total_boxes] (sample-major order).\"\"\"\n", + "\n", + " def __init__(self, num_classes, grid_size):\n", + " super().__init__()\n", + " self.num_classes = num_classes\n", + " self.grid_size = grid_size\n", + "\n", + " def forward(self, outputs, targets):\n", + " per_sample = _per_box_iou(outputs, targets, self.grid_size)\n", + " flat = [v for s in per_sample for v in s]\n", + " if not flat:\n", + " return torch.zeros(0, device=outputs.device)\n", + " return torch.stack(flat).detach()\n", + "\n", + "\n", + "# =============================================================================\n", + "# Inference-time decoding (for UI prediction overlays)\n", + "# =============================================================================\n", + "def decode_predictions(outputs, grid_size, conf_thresh=0.3, max_det=10):\n", + " \"\"\"Turn raw grid logits into a per-sample list of detected boxes.\n", + "\n", + " Returns list[B] of [M, 6] numpy-friendly tensors\n", + " ``[x1, y1, x2, y2, class_id, confidence]`` (kept on CPU, detached) — the\n", + " exact 6-column schema WL renders for detection predictions.\n", + " \"\"\"\n", + " boxes_grid, obj, cls_probs = decode_grid(outputs, grid_size)\n", + " B, S = outputs.shape[0], grid_size\n", + "\n", + " cls_conf, cls_id = cls_probs.max(dim=-1) # [B, S, S]\n", + " score = obj * cls_conf # combined confidence\n", + "\n", + " flat_boxes = boxes_grid.view(B, S * S, 4)\n", + " flat_score = score.view(B, S * S)\n", + " flat_cls = cls_id.view(B, S * S)\n", + "\n", + " results = []\n", + " for s in range(B):\n", + " keep = flat_score[s] >= conf_thresh\n", + " if keep.sum() == 0:\n", + " results.append(torch.zeros((0, 6)))\n", + " continue\n", + " sc = flat_score[s][keep]\n", + " bx = flat_boxes[s][keep]\n", + " cl = flat_cls[s][keep].to(bx.dtype)\n", + "\n", + " # Keep the most confident detections (cheap top-k in place of full NMS).\n", + " order = torch.argsort(sc, descending=True)[:max_det]\n", + " det = torch.cat([bx[order], cl[order, None], sc[order, None]], dim=1)\n", + " results.append(det.detach().cpu())\n", + "\n", + " return results" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9hjJ9fHD_xBf" + }, + "source": [ + "## 3. Configuration\n", + "\n", + "Every tunable lives here in one dict (the example's `config.yaml`, inlined with its comments). Wrapping it with `flag=\"hyperparameters\"` lets the Studio UI read - and live-edit - these values while training." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "YZCPlGjN_xBf", + "outputId": "97c94163-c64d-4f7e-8ce7-d812b5983885", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:32:57.025 INFO:weightslab.components.checkpoint_manager:__init__: CheckpointManager initialized at /tmp/tmpf35sd4_z\n", + "16/07/2026-08:32:57.030 INFO:weightslab.src:watch_or_edit: Registered new checkpoint manager in ledger\n", + "16/07/2026-08:32:57.040 INFO:root:set_log_directory: Log file moved from /tmp/tmpmt3c2rur/weightslab_20260716_083240.log to /tmp/tmpf35sd4_z/weightslab_20260716_083240.log\n", + "16/07/2026-08:32:57.041 INFO:root:set_log_directory: Log directory updated to: /tmp/tmpf35sd4_z\n", + "16/07/2026-08:32:57.044 INFO:root:set_log_directory: Log file: /tmp/tmpf35sd4_z/weightslab_20260716_083240.log\n", + "Experiment logs -> /tmp/tmpf35sd4_z\n" + ] + } + ], + "source": [ + "# Inlined from the example's config.yaml (comments preserved).\n", + "config = {\n", + " # -- Global -----------------------------------------------------------\n", + " \"device\": \"auto\", # auto -> cuda if available else cpu (resolved in the imports cell)\n", + " \"experiment_name\": \"detection_pennfudan_usecase\", # name shown in Weights Studio\n", + " \"training_steps_to_do\": None, # None = open-ended in the example; capped below for Colab\n", + " \"root_log_dir\": None, # None -> a temp dir is created below\n", + "\n", + " \"checkpoint_manager\": {\n", + " \"load_config\": False, # ignore a previous run's config so edits here take effect\n", + " },\n", + "\n", + " # Initially compute natural-sort signals, e.g. average intensity.\n", + " \"compute_natural_sort\": True,\n", + "\n", + " # -- Experiment schedule ---------------------------------------------\n", + " \"eval_full_to_train_steps_ratio\": 5, # run a full eval every N steps\n", + " \"experiment_dump_to_train_steps_ratio\": 5, # export history + dataframe every N steps\n", + " \"tqdm_display\": True,\n", + " \"is_training\": False, # start training immediately or not\n", + "\n", + " # -- Serving ----------------------------------------------------------\n", + " \"serving_grpc\": True, # expose the gRPC backend for Weights Studio\n", + " \"serving_cli\": True,\n", + "\n", + " # -- Global dataframe storage ----------------------------------------\n", + " \"ledger_enable_h5_persistence\": True,\n", + " \"ledger_enable_flushing_threads\": True,\n", + " \"ledger_flush_max_rows\": 100,\n", + " \"ledger_flush_interval\": 60.0,\n", + "\n", + " # -- Model / task -----------------------------------------------------\n", + " \"num_classes\": 1, # Penn-Fudan: single class (person)\n", + " \"image_size\": 128,\n", + " \"grid_size\": 8, # detector predicts on an 8x8 cell grid\n", + " \"conf_thresh\": 0.3, # objectness*class confidence threshold for displayed predictions\n", + " \"pretrained_backbone\": True, # ImageNet-pretrained MobileNetV3-Small feature extractor\n", + " \"freeze_backbone\": True, # train only the detection head (faster, less data hungry)\n", + "\n", + " \"optimizer\": {\n", + " \"lr\": 0.001,\n", + " },\n", + "\n", + " # -- Data (Penn-Fudan pedestrians; ~170 images downloaded on first run under data_root) --\n", + " \"data_root\": \"./data\", # portable relative path; the dataset downloads here on first run\n", + " \"data\": {\n", + " \"train_loader\": {\n", + " \"batch_size\": 8,\n", + " \"shuffle\": True,\n", + " \"max_samples\": None, # None = use the full training split\n", + " },\n", + " \"test_loader\": {\n", + " \"batch_size\": 8,\n", + " \"shuffle\": False,\n", + " \"max_samples\": None,\n", + " },\n", + " },\n", + "}\n", + "\n", + "wl.watch_or_edit(config, flag=\"hyperparameters\", defaults=config, poll_interval=1.0)\n", + "\n", + "# Resolve the log directory (a temp dir when unset), mirroring the example's main.py.\n", + "log_dir = config.get(\"root_log_dir\") or tempfile.mkdtemp(prefix=\"weightslab_detection_\")\n", + "config[\"root_log_dir\"] = log_dir\n", + "os.makedirs(log_dir, exist_ok=True)\n", + "print(\"Experiment logs ->\", log_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vWWllcOA_xBg" + }, + "source": [ + "## 4. Data\n", + "\n", + "Build the Penn-Fudan train/val datasets (the ~170 images **download automatically** on first construction) and wrap them as tracked dataloaders. Detection targets have a variable number of boxes per image, so the custom `det_collate` keeps them as a list." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "Xqkj3Lo__xBg", + "outputId": "ae81a09b-c057-45d3-e3b3-254d99b31e78", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[data] Downloading Penn-Fudan dataset to ./data/PennFudanPed.zip ...\n", + "[data] Extracting Penn-Fudan ...\n", + "16/07/2026-08:33:02.465 INFO:weightslab.data.data_samples_with_ops:__init__: [DataSampleTrackingWrapper] H5 persistence enabled at /tmp/tmpf35sd4_z/checkpoints/data/data.h5\n", + "16/07/2026-08:33:02.469 INFO:weightslab.data.data_samples_with_ops:__init__: Preloading sample statistics for PHYSICAL indices in split 'train_loader' with: preload_metadata=True...\n", + "16/07/2026-08:33:02.471 INFO:weightslab.data.data_samples_with_ops:__init__: Labels will be loaded on demand from the wrapped dataset for split 'train_loader' when accessed, which may increase latency on first access but reduces initialization time.\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "Initializing ledger for split 'train_loader': 100%|██████████| 136/136 [00:00<00:00, 94114.06it/s]" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:33:02.481 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] Registering split 'train_loader' with 136 samples.\n", + "16/07/2026-08:33:02.527 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] After annotation expansion: 136 samples → 136 annotation rows.\n", + "16/07/2026-08:33:02.528 INFO:weightslab.data.h5_array_store:__init__: [H5ArrayStore] Initialized with cache limit: 2048MB\n", + "16/07/2026-08:33:02.562 INFO:weightslab.data.data_samples_with_ops:__init__: [DataSampleTrackingWrapper] H5 persistence enabled at /tmp/tmpf35sd4_z/checkpoints/data/data.h5\n", + "16/07/2026-08:33:02.564 INFO:weightslab.data.data_samples_with_ops:__init__: Preloading sample statistics for PHYSICAL indices in split 'test_loader' with: preload_labels=True, preload_metadata=True...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Initializing ledger for split 'test_loader': 100%|██████████| 34/34 [00:00<00:00, 102.86it/s]" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:33:02.902 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] Registering split 'test_loader' with 34 samples.\n", + "16/07/2026-08:33:02.907 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] After annotation expansion: 34 samples → 34 annotation rows.\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Penn-Fudan auto-downloads (~170 images) on first dataset construction - nothing to upload.\n", + "data_root = config.get(\"data_root\") or \"./data\"\n", + "os.makedirs(data_root, exist_ok=True)\n", + "\n", + "num_classes = int(config[\"num_classes\"])\n", + "image_size = int(config[\"image_size\"])\n", + "grid_size = int(config[\"grid_size\"])\n", + "conf_thresh = float(config[\"conf_thresh\"])\n", + "\n", + "train_cfg = config.get(\"data\", {}).get(\"train_loader\", {})\n", + "test_cfg = config.get(\"data\", {}).get(\"test_loader\", {})\n", + "\n", + "_train_dataset = PennFudanDetectionDataset(\n", + " root=data_root,\n", + " split=\"train\",\n", + " num_classes=num_classes,\n", + " image_size=image_size,\n", + " max_samples=train_cfg.get(\"max_samples\", None),\n", + ")\n", + "_val_dataset = PennFudanDetectionDataset(\n", + " root=data_root,\n", + " split=\"val\",\n", + " num_classes=num_classes,\n", + " image_size=image_size,\n", + " max_samples=test_cfg.get(\"max_samples\", None),\n", + ")\n", + "\n", + "# Tracked dataloaders (detection uses the custom det_collate defined above).\n", + "train_loader = wl.watch_or_edit(\n", + " _train_dataset,\n", + " flag=\"data\",\n", + " loader_name=\"train_loader\",\n", + " batch_size=train_cfg.get(\"batch_size\", 8),\n", + " shuffle=train_cfg.get(\"shuffle\", True),\n", + " compute_hash=False,\n", + " is_training=True,\n", + " array_autoload_arrays=False,\n", + " array_return_proxies=True,\n", + " array_use_cache=True,\n", + " preload_labels=False,\n", + " collate_fn=det_collate,\n", + ")\n", + "test_loader = wl.watch_or_edit(\n", + " _val_dataset,\n", + " flag=\"data\",\n", + " loader_name=\"test_loader\",\n", + " batch_size=test_cfg.get(\"batch_size\", 8),\n", + " shuffle=test_cfg.get(\"shuffle\", False),\n", + " compute_hash=False,\n", + " is_training=False,\n", + " array_autoload_arrays=False,\n", + " array_return_proxies=True,\n", + " array_use_cache=True,\n", + " preload_labels=True,\n", + " collate_fn=det_collate,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pvj_ywPr_xBg" + }, + "source": [ + "## 5. Model, optimizer and tracked signals\n", + "\n", + "Wrap the detector, optimizer, loss and IoU metrics with `wl.watch_or_edit(...)`. The loss is tracked **per sample**; per-instance IoU stores one value per ground-truth box at `(sample_id, annotation_id)`." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "21SKj-oK_xBg", + "outputId": "698a493f-d5a5-4e42-c399-563b6addc2bf", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading: \"https://download.pytorch.org/models/mobilenet_v3_small-047dcff4.pth\" to /root/.cache/torch/hub/checkpoints/mobilenet_v3_small-047dcff4.pth\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "100%|██████████| 9.83M/9.83M [00:00<00:00, 133MB/s]" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:33:23.329 INFO:weightslab.backend.model_interface:__init__: Using checkpoint manager from ledger\n", + "\n", + "============================================================\n", + "Computing class weights for 1 classes (max 200 samples)...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + " Analyzing Distribution: 100%|██████████| 136/136 [00:00<00:00, 147.35it/s]" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\n", + "Class distribution and weights:\n", + "Class 0 (person): 100.00% -> weight: 1.000\n", + "============================================================\n", + "\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "/tmp/ipykernel_5133/4133204388.py:171: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " \"weights\", torch.tensor(weights) if weights is not None else None\n" + ] + } + ], + "source": [ + "# --- Model ---\n", + "_model = SmallDetector(\n", + " in_channels=3, num_classes=num_classes,\n", + " image_size=image_size, grid_size=grid_size,\n", + " pretrained=bool(config[\"pretrained_backbone\"]),\n", + " freeze_backbone=bool(config[\"freeze_backbone\"]),\n", + ").to(device)\n", + "model = wl.watch_or_edit(_model, flag=\"model\", device=device)\n", + "\n", + "# --- Optimizer (only trainable params; the backbone may be frozen) ---\n", + "lr = config.get(\"optimizer\", {}).get(\"lr\", 1e-3)\n", + "trainable = [p for p in model.parameters() if p.requires_grad]\n", + "_optimizer = optim.Adam(trainable, lr=lr)\n", + "optimizer = wl.watch_or_edit(_optimizer, flag=\"optimizer\")\n", + "\n", + "\n", + "# --- Detection loss (per sample) + IoU (per sample & per instance) signals ---\n", + "# per_sample=True auto-saves one value per sample; per_instance=True auto-saves\n", + "# one IoU per (sample_id, annotation_id), i.e. one per ground-truth box.\n", + "def _make_det_signals(split: str, weights=None) -> dict:\n", + " return {\n", + " \"loss\": wl.watch_or_edit(\n", + " PerSampleDetectionLoss(num_classes, grid_size, weights=weights),\n", + " flag=\"loss\",\n", + " name=f\"{split}_loss/sample\", per_sample=True, log=True,\n", + " ),\n", + " \"iou_sample\": wl.watch_or_edit(\n", + " PerSampleIoU(num_classes, grid_size), flag=\"metric\",\n", + " name=f\"{split}_iou/sample\", per_sample=True, log=True,\n", + " ),\n", + " \"iou_instance\": wl.watch_or_edit(\n", + " PerInstanceIoU(num_classes, grid_size), flag=\"metric\",\n", + " name=f\"{split}_iou/instance\", per_instance=True, log=True,\n", + " ),\n", + " }\n", + "\n", + "\n", + "# Class weights from ground-truth box counts (optional; balances rare shapes).\n", + "def compute_class_weights(dataset, num_classes, max_samples=200):\n", + " print(\"\\n\" + \"=\" * 60, flush=True)\n", + " print(f\"Computing class weights for {num_classes} classes (max {max_samples} samples)...\", flush=True)\n", + " class_counts = np.zeros(num_classes, dtype=np.float64)\n", + " num_samples = min(len(dataset), max_samples)\n", + "\n", + " for idx in tqdm.tqdm(range(num_samples), desc=\" Analyzing Distribution\"):\n", + " _, _, target, _ = dataset.get_items(idx, include_labels=True)\n", + " if target is None or len(target) == 0:\n", + " continue\n", + " for c in target[:, 4].astype(np.int64):\n", + " if 0 <= c < num_classes:\n", + " class_counts[c] += 1\n", + "\n", + " class_counts = np.maximum(class_counts, 1) # Avoid div by zero\n", + " total = class_counts.sum()\n", + " class_weights = total / (num_classes * class_counts)\n", + " class_weights = class_weights / class_weights.mean() # Normalize\n", + "\n", + " print(\"\\nClass distribution and weights:\", flush=True)\n", + " for c in range(num_classes):\n", + " pct = (class_counts[c] / total) * 100\n", + " print(f\"Class {c} ({dataset.class_names[c]}): {pct:6.2f}% -> weight: {class_weights[c]:.3f}\", flush=True)\n", + " print(\"=\" * 60 + \"\\n\", flush=True)\n", + " return torch.FloatTensor(class_weights).to(device)\n", + "\n", + "\n", + "weights = compute_class_weights(_train_dataset, num_classes)\n", + "\n", + "train_sig = _make_det_signals(\"train\", weights=weights)\n", + "test_sig = _make_det_signals(\"test\", weights=weights)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xjCJ2AYV_xBg" + }, + "source": [ + "## 6. Train and evaluate steps\n", + "\n", + "The `guard_training_context` / `guard_testing_context` blocks tell WeightsLab which phase it's in. The watched loss rides along with `batch_ids=ids` (so it is stored per sample) and `preds=` (so the decoded boxes are saved for the UI overlay)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "id": "g_m26pWr_xBg" + }, + "outputs": [], + "source": [ + "def train(loader, model, optimizer, sig, device, grid_size, conf_thresh):\n", + " \"\"\"Single training step using the tracked dataloader + watched loss.\n", + "\n", + " loader yields (inputs, ids, targets, metadata) because of the\n", + " DataSampleTrackingWrapper. `targets` is per sample a [N, 6] tensor of boxes\n", + " ([x1, y1, x2, y2, class_id, confidence]); see utils/data.det_collate.\n", + " \"\"\"\n", + " with guard_training_context:\n", + " (inputs, ids, targets, _) = next(loader)\n", + " inputs = inputs.to(device)\n", + " targets = [t.to(device) for t in targets]\n", + "\n", + " optimizer.zero_grad()\n", + " outputs = model(inputs) # [B, S, S, 5 + num_classes]\n", + "\n", + " # Decoded boxes for the UI overlay (detached - display only).\n", + " preds = decode_predictions(outputs.detach(), grid_size, conf_thresh=conf_thresh)\n", + "\n", + " # Per-sample detection loss (tracked, saved, and the value we backprop).\n", + " # `preds=` rides along so WL stores the predicted boxes for this batch.\n", + " loss_per_sample = sig[\"loss\"](outputs, targets, batch_ids=ids, preds=preds)\n", + "\n", + " # Metrics: per-sample mean IoU + per-instance IoU (one value per box).\n", + " sig[\"iou_sample\"](outputs, targets, batch_ids=ids)\n", + " sig[\"iou_instance\"](outputs, targets, batch_ids=ids)\n", + "\n", + " loss = loss_per_sample.mean()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " return float(loss.detach().cpu().item())\n", + "\n", + "\n", + "def test(loader, model, sig, device, grid_size, conf_thresh, test_loader_len):\n", + " \"\"\"Full evaluation pass over the val loader.\"\"\"\n", + " losses = 0.0\n", + " ious = 0.0\n", + " with guard_testing_context, torch.no_grad():\n", + " for inputs, ids, targets, _ in loader:\n", + " inputs = inputs.to(device)\n", + " targets = [t.to(device) for t in targets]\n", + "\n", + " outputs = model(inputs)\n", + " preds = decode_predictions(outputs, grid_size, conf_thresh=conf_thresh)\n", + "\n", + " loss_per_sample = sig[\"loss\"](outputs, targets, batch_ids=ids, preds=preds)\n", + " iou_per_sample = sig[\"iou_sample\"](outputs, targets, batch_ids=ids)\n", + " sig[\"iou_instance\"](outputs, targets, batch_ids=ids)\n", + "\n", + " losses += torch.mean(loss_per_sample)\n", + " ious += torch.mean(iou_per_sample)\n", + "\n", + " loss = float((losses / test_loader_len).detach().cpu().item())\n", + " iou = float((ious / test_loader_len).detach().cpu().item())\n", + " return loss, iou * 100.0 # Return mean IoU as percentage" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QsFSIK3j_xBg" + }, + "source": [ + "## 7. Serve and train\n", + "\n", + "`wl.serve(...)` starts the background gRPC server (and a public `bore` tunnel) that Weights Studio connects to. `wl.start_training(...)` flips the experiment into the *training* state, then we run the loop, periodically evaluating and exporting signals.\n", + "\n", + "Leave this cell **running** while you watch it stream in the UI." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5yQOd8iv_xBg", + "outputId": "19c6f02b-80b3-4eea-a207-aeef7c83d6f5", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:33:32.039 INFO:weightslab.trainer.trainer_services:_run_security_preflight: \n", + "\n", + "# #######################################\n", + "# #######################################\n", + "[gRPC] Security preflight checks:\n", + "\tTLS: DISABLED (unencrypted traffic)\n", + "\tAuth tokens: NONE configured\n", + "\t! WARNING: GRPC_TLS_ENABLED=0. Traffic will be unencrypted. Use only for development.\n", + "\t! WARNING: No GRPC_AUTH_TOKEN/GRPC_AUTH_TOKENS configured. Only transport-level trust (TLS/mTLS) will protect RPC access.\n", + "# #######################################\n", + "# #######################################\n", + "\n", + "16/07/2026-08:33:32.041 INFO:weightslab.trainer.trainer_services:grpc_serve: [gRPC] Watchdogs disabled via WEIGHTSLAB_DISABLE_WATCHDOGS.\n", + "16/07/2026-08:33:32.045 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Thread callback started\n", + "16/07/2026-08:33:32.045 INFO:weightslab.trainer.trainer_services:grpc_serve: [gRPC] Server started with watchdogs disabled (host=0.0.0.0 port=50051 workers=None)\n", + "16/07/2026-08:33:32.047 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Creating ThreadPoolExecutor with 6 worker threads (n_workers_grpc=None, max_concurrent_rpcs=None)\n", + "16/07/2026-08:33:32.051 INFO:weightslab.tunnel:_ensure_bore: Downloading bore v0.6.0 (bore-v0.6.0-x86_64-unknown-linux-musl.tar.gz)...\n", + "16/07/2026-08:33:32.129 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Server object created\n", + "16/07/2026-08:33:32.142 INFO:weightslab.trainer.services.agent.agent:__init__: Initializing DataManipulationAgent\n", + "16/07/2026-08:33:32.156 WARNING:weightslab.trainer.services.agent.agent:_load_config: Error loading config from /usr/local/lib/python3.12/dist-packages: [Errno 21] Is a directory: '/usr/local/lib/python3.12/dist-packages'\n", + "16/07/2026-08:33:32.157 INFO:weightslab.trainer.services.agent.agent:_load_config: \n", + "\n", + "# #######################################\n", + "# #######################################\n", + "Agent initialized from configuration /content/agent_config.yaml: \n", + "\tFinal Agent Configuration: Preferred Provider=openrouter, \n", + "\tFallback to Local=True, \n", + "\tOpenRouter Model=~google/gemini-flash-latest with:\n", + "\t\tAPI Key=None\n", + "\t\tBase URL=https://openrouter.ai/api/v1, \n", + "\tOllama Model=llama3.2:3b\n", + "# #######################################\n", + "# #######################################\n", + "\n", + "16/07/2026-08:33:32.158 INFO:weightslab.trainer.services.agent.agent:_setup_providers: Setting up Ollama with model llama3.2:3b\n", + "16/07/2026-08:33:32.260 INFO:weightslab.trainer.services.agent.agent:_setup_providers: [Agent] Ollama enabled: llama3.2:3b\n", + "16/07/2026-08:33:32.261 INFO:weightslab.trainer.services.data_service:_compute_natural_sort_stats: [DataService] Starting natural sort stats computation with weights: {'brightness': 0.7, 'entropy': 0.3, 'hue': 0.0}\n", + "16/07/2026-08:33:32.273 INFO:weightslab.trainer.services.data_service:_compute_natural_sort_stats: [DataService] Computing sort stats for 170 samples...\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "Computing Natural Sort Stats: 26%|██▌ | 44/170 [00:00<00:02, 44.78samples/s]" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "============================================================\n", + " Backend exposed via bore at: bore.pub:41162\n", + " On your local machine (Docker running), run:\n", + " weightslab start\n", + " weightslab tunnel bore.pub:41162\n", + "============================================================\n", + "16/07/2026-08:33:33.256 INFO:weightslab.backend.cli:cli_serve: cli_thread_started\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\rComputing Natural Sort Stats: 31%|███ | 53/170 [00:00<00:02, 54.32samples/s]" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:33:33.346 INFO:weightslab.src:start_training: Starting WeightsLab training mode with a timeout of 3 seconds.\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "Computing Natural Sort Stats: 100%|██████████| 170/170 [00:02<00:00, 62.04samples/s]" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:33:35.145 INFO:weightslab.trainer.services.data_service:_compute_natural_sort_stats: [DataService] Completed stats computation for 170 samples\n", + "\n", + "\n", + "Natural sort computation finished for 170 samples\n", + "\n", + "\n", + "16/07/2026-08:33:35.147 INFO:weightslab.trainer.services.data_service:_build_preview_cache: [PreviewCache] Building 64×64 or less or less preview cache for 170 samples …\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "/usr/local/lib/python3.12/dist-packages/weightslab/data/dataframe_manager.py:703: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '[7.64467454 7.63759617 7.82511253 7.60454149 7.68511861 7.70962498\n", + " 7.66256563 7.67420456 7.49258183 7.81929857 6.45717949 7.54281418\n", + " 6.79016624 7.48513405 7.32040805 7.82787458 7.77316711 7.5562755\n", + " 7.61231887 7.63537192 7.82628297 7.76547166 7.13113334 7.52482093\n", + " 7.63815989 7.7459368 7.58974044 7.75762173 7.18705444 7.48240921\n", + " 7.76600207 7.76543333 7.71791639 7.79005573]' has dtype incompatible with float32, please explicitly cast to a compatible dtype first.\n", + " self._df.loc[existing_idx, all_cols] = df_norm.loc[existing_idx, all_cols]\n", + "/usr/local/lib/python3.12/dist-packages/weightslab/data/dataframe_manager.py:703: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '[ 53.51681774 93.30201817 61.98435004 65.24323962 43.56923658\n", + " 93.96657441 67.03601995 73.99117037 80.06768645 84.95401979\n", + " 88.10120384 55.8291746 82.3123762 58.22935624 100.60901833\n", + " 94.05543837 79.249868 53.99237577 65.56806674 73.41094873\n", + " 99.16436384 101.13093832 93.03205657 99.69871153 101.89878035\n", + " 85.80537355 86.84879743 86.214422 99.49009009 81.44740104\n", + " 56.29921017 61.5624775 67.23737161 66.83788963]' has dtype incompatible with float32, please explicitly cast to a compatible dtype first.\n", + " self._df.loc[existing_idx, all_cols] = df_norm.loc[existing_idx, all_cols]\n", + "/usr/local/lib/python3.12/dist-packages/weightslab/data/dataframe_manager.py:703: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '[ 64.49066497 65.37024572 104.59268596 52.93642734 72.02098046\n", + " 48.14319779 113.24922957 46.29628522 43.88088143 25.53077158\n", + " 60.70043395 49.99961312 32.48544889 87.93221913 36.88762984\n", + " 47.55134406 57.05124766 64.13736411 65.30893805 96.26249138\n", + " 38.45269062 42.4448374 35.35399698 49.62630278 46.2604104\n", + " 42.64606061 32.72595598 68.50939554 46.54847017 59.36240918\n", + " 55.71851252 58.31763177 50.1109307 61.99646905]' has dtype incompatible with float32, please explicitly cast to a compatible dtype first.\n", + " self._df.loc[existing_idx, all_cols] = df_norm.loc[existing_idx, all_cols]\n", + "/usr/local/lib/python3.12/dist-packages/weightslab/data/dataframe_manager.py:703: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '[0.65953828 0.61511859 0.63585766 0.62043339 0.63605741 0.58398832\n", + " 0.53613578 0.62181845 0.62159035 0.66207224 0.38428878 0.64687088\n", + " 0.52937324 0.5747199 0.54391107 0.61459091 0.61792182 0.55884669\n", + " 0.5585666 0.58468377 0.60382653 0.58175791 0.50977316 0.59231942\n", + " 0.5993173 0.63587023 0.63463003 0.58706704 0.4947014 0.59870562\n", + " 0.58584364 0.62449927 0.61637045 0.59844273]' has dtype incompatible with float32, please explicitly cast to a compatible dtype first.\n", + " self._df.loc[existing_idx, all_cols] = df_norm.loc[existing_idx, all_cols]\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:33:35.148 INFO:weightslab.trainer.services.data_service:__init__: DataService initialized.\n", + "16/07/2026-08:33:35.150 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Servicer added\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\r[PreviewCache]: 0%| | 0/170 [00:00 16 (Loader: train_loader)\n", + "16/07/2026-08:40:17.818 INFO:weightslab.components.global_monitoring:resume: \n", + "Training resumed as modules hashes have been computed: ['bfb454d6', 'd39e7bd9', '43e44972'].\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n", + "\n", + "Training: 20%|█▉ | 296/1500 [06:41<33:32, 1.67s/it, iou=56.44%, test_loss=8.1024, train_loss=1.8979]\u001b[A\n", + "Training: 20%|█▉ | 297/1500 [06:41<10:23:18, 31.09s/it, iou=56.44%, test_loss=8.1024, train_loss=1.8979]\u001b[A\n", + "Training: 20%|█▉ | 297/1500 [06:41<10:23:18, 31.09s/it, iou=56.44%, test_loss=8.1024, train_loss=2.4030]\u001b[A\n", + "Training: 20%|█▉ | 298/1500 [06:41<7:18:34, 21.89s/it, iou=56.44%, test_loss=8.1024, train_loss=2.4030] \u001b[A\n", + "Training: 20%|█▉ | 298/1500 [06:42<7:18:34, 21.89s/it, iou=56.44%, test_loss=8.1024, train_loss=1.9995]\u001b[A\n", + "Training: 20%|█▉ | 299/1500 [06:42<5:11:45, 15.57s/it, iou=56.44%, test_loss=8.1024, train_loss=1.9995]\u001b[A\n", + "Training: 20%|█▉ | 299/1500 [06:42<5:11:45, 15.57s/it, iou=56.44%, test_loss=8.1024, train_loss=1.9483]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:40:20.402 INFO:weightslab.components.checkpoint_manager:save_model_checkpoint: Saved model checkpoint: bfb454d6d39e7bd943e44972_step_000300.pt\n", + "16/07/2026-08:40:20.588 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmpf35sd4_z/checkpoints/loggers/loggers.manifest.json (1 chunks)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 20%|█▉ | 299/1500 [06:43<5:11:45, 15.57s/it, iou=56.44%, test_loss=8.1024, train_loss=2.3940]\u001b[A\n", + "Training: 20%|██ | 301/1500 [06:43<2:51:19, 8.57s/it, iou=56.44%, test_loss=8.1024, train_loss=2.3940]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:40:21.165 INFO:weightslab.backend.dataloader_interface:set_batch_size: Batch size updated: 8 -> 16 (Loader: test_loader)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:40:23.499 INFO:weightslab.src:write_dataframe: write_dataframe: wrote 593 row(s) × 24 column(s) as json to /tmp/tmpf35sd4_z/d8e01c87_dataframe.json\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 20%|██ | 301/1500 [06:46<2:51:19, 8.57s/it, iou=33.90%, test_loss=4.7464, train_loss=1.9816]\u001b[A\n", + "Training: 20%|██ | 302/1500 [06:46<2:22:34, 7.14s/it, iou=33.90%, test_loss=4.7464, train_loss=1.9816]\u001b[A\n", + "Training: 20%|██ | 302/1500 [06:46<2:22:34, 7.14s/it, iou=33.90%, test_loss=4.7464, train_loss=1.8739]\u001b[A\n", + "Training: 20%|██ | 303/1500 [06:46<1:47:37, 5.39s/it, iou=33.90%, test_loss=4.7464, train_loss=1.8739]\u001b[A\n", + "Training: 20%|██ | 303/1500 [06:46<1:47:37, 5.39s/it, iou=33.90%, test_loss=4.7464, train_loss=1.8943]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n", + "\n", + "Training: 20%|██ | 303/1500 [06:47<1:47:37, 5.39s/it, iou=33.90%, test_loss=4.7464, train_loss=1.8743]\u001b[A\n", + "Training: 20%|██ | 305/1500 [06:47<1:03:34, 3.19s/it, iou=33.90%, test_loss=4.7464, train_loss=1.8743]\u001b[A\n", + "Training: 20%|██ | 305/1500 [06:47<1:03:34, 3.19s/it, iou=33.90%, test_loss=4.7464, train_loss=1.9137]\u001b[A\n", + "Training: 20%|██ | 306/1500 [06:47<50:33, 2.54s/it, iou=33.90%, test_loss=4.7464, train_loss=1.9137] \u001b[A\n", + "Training: 20%|██ | 306/1500 [06:48<50:33, 2.54s/it, iou=33.90%, test_loss=4.7464, train_loss=2.4774]\u001b[A\n", + "Training: 20%|██ | 307/1500 [06:48<42:26, 2.13s/it, iou=33.90%, test_loss=4.7464, train_loss=2.4774]\u001b[A\n", + "Training: 20%|██ | 307/1500 [06:48<42:26, 2.13s/it, iou=33.90%, test_loss=4.7464, train_loss=1.8858]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n", + "\n", + "Training: 20%|██ | 307/1500 [06:49<42:26, 2.13s/it, iou=33.90%, test_loss=4.7464, train_loss=1.9598]\u001b[A\n", + "Training: 21%|██ | 309/1500 [06:49<26:52, 1.35s/it, iou=33.90%, test_loss=4.7464, train_loss=1.9598]\u001b[A\n", + "Training: 21%|██ | 309/1500 [06:49<26:52, 1.35s/it, iou=33.90%, test_loss=4.7464, train_loss=2.3849]\u001b[A\n", + "Training: 21%|██ | 310/1500 [06:49<22:41, 1.14s/it, iou=33.90%, test_loss=4.7464, train_loss=2.3849]\u001b[A\n", + "Training: 21%|██ | 310/1500 [06:50<22:41, 1.14s/it, iou=33.90%, test_loss=4.7464, train_loss=1.8953]\u001b[A\n", + "Training: 21%|██ | 311/1500 [06:50<21:04, 1.06s/it, iou=33.90%, test_loss=4.7464, train_loss=1.8953]\u001b[A\n", + "Training: 21%|██ | 311/1500 [06:50<21:04, 1.06s/it, iou=33.90%, test_loss=4.7464, train_loss=1.9958]\u001b[A\n", + "Training: 21%|██ | 312/1500 [06:50<16:32, 1.20it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9958]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n", + "\n", + "Training: 21%|██ | 312/1500 [06:51<16:32, 1.20it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8536]\u001b[A\n", + "Training: 21%|██ | 313/1500 [06:51<14:17, 1.38it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8536]\u001b[A\n", + "Training: 21%|██ | 313/1500 [06:51<14:17, 1.38it/s, iou=33.90%, test_loss=4.7464, train_loss=2.3955]\u001b[A\n", + "Training: 21%|██ | 314/1500 [06:51<13:25, 1.47it/s, iou=33.90%, test_loss=4.7464, train_loss=2.3955]\u001b[A\n", + "Training: 21%|██ | 314/1500 [06:52<13:25, 1.47it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9691]\u001b[A\n", + "Training: 21%|██ | 315/1500 [06:52<16:04, 1.23it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9691]\u001b[A\n", + "Training: 21%|██ | 315/1500 [06:53<16:04, 1.23it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8806]\u001b[A\n", + "Training: 21%|██ | 316/1500 [06:53<13:43, 1.44it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8806]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n", + "\n", + "Training: 21%|██ | 316/1500 [06:54<13:43, 1.44it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8700]\u001b[A\n", + "Training: 21%|██ | 317/1500 [06:54<17:55, 1.10it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8700]\u001b[A\n", + "Training: 21%|██ | 317/1500 [06:55<17:55, 1.10it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9674]\u001b[A\n", + "Training: 21%|██ | 318/1500 [06:55<16:26, 1.20it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9674]\u001b[A\n", + "Training: 21%|██ | 318/1500 [06:56<16:26, 1.20it/s, iou=33.90%, test_loss=4.7464, train_loss=2.3859]\u001b[A\n", + "Training: 21%|██▏ | 319/1500 [06:56<18:06, 1.09it/s, iou=33.90%, test_loss=4.7464, train_loss=2.3859]\u001b[A\n", + "Training: 21%|██▏ | 319/1500 [06:56<18:06, 1.09it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8463]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n", + "\n", + "Training: 21%|██▏ | 319/1500 [06:56<18:06, 1.09it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9668]\u001b[A\n", + "Training: 21%|██▏ | 321/1500 [06:56<11:56, 1.64it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9668]\u001b[A\n", + "Training: 21%|██▏ | 321/1500 [06:57<11:56, 1.64it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8672]\u001b[A\n", + "Training: 21%|██▏ | 322/1500 [06:57<11:02, 1.78it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8672]\u001b[A\n", + "Training: 21%|██▏ | 322/1500 [06:58<11:02, 1.78it/s, iou=33.90%, test_loss=4.7464, train_loss=2.3978]\u001b[A\n", + "Training: 22%|██▏ | 323/1500 [06:58<12:20, 1.59it/s, iou=33.90%, test_loss=4.7464, train_loss=2.3978]\u001b[A\n", + "Training: 22%|██▏ | 323/1500 [06:58<12:20, 1.59it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9003]\u001b[A\n", + "Training: 22%|██▏ | 324/1500 [06:58<09:58, 1.96it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9003]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n", + "\n", + "Training: 22%|██▏ | 324/1500 [06:58<09:58, 1.96it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8884]\u001b[A\n", + "Training: 22%|██▏ | 325/1500 [06:58<09:27, 2.07it/s, iou=33.90%, test_loss=4.7464, train_loss=1.8884]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:40:36.225 INFO:weightslab.components.checkpoint_manager:save_model_checkpoint: Saved model checkpoint: bfb454d6d39e7bd943e44972_step_000325.pt\n", + "16/07/2026-08:40:36.404 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmpf35sd4_z/checkpoints/loggers/loggers.manifest.json (1 chunks)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 22%|██▏ | 325/1500 [06:59<09:27, 2.07it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9650]\u001b[A\n", + "Training: 22%|██▏ | 326/1500 [06:59<10:33, 1.85it/s, iou=33.90%, test_loss=4.7464, train_loss=1.9650]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:40:38.855 INFO:weightslab.components.global_monitoring:pause: \n", + "Training paused.\n", + "16/07/2026-08:40:38.862 INFO:weightslab.trainer.services.experiment_service:ExperimentCommand: [WeightsLab] UI Command: HP changed, experiment paused!\n", + "16/07/2026-08:40:38.865 INFO:weightslab.trainer.services.experiment_service:ExperimentCommand: \n", + "[WeightsLab] UI Command: PAUSE\n", + "16/07/2026-08:40:38.870 INFO:weightslab.components.global_monitoring:pause: \n", + "Training paused.\n", + "16/07/2026-08:40:39.412 INFO:weightslab.src:write_dataframe: write_dataframe: wrote 593 row(s) × 24 column(s) as json to /tmp/tmpf35sd4_z/d8e01c87_dataframe.json\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 22%|██▏ | 326/1500 [07:02<10:33, 1.85it/s, iou=54.03%, test_loss=7.9861, train_loss=2.4054]\u001b[A\n", + "Training: 22%|██▏ | 327/1500 [07:02<23:56, 1.22s/it, iou=54.03%, test_loss=7.9861, train_loss=2.4054]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:40:41.095 INFO:weightslab.trainer.services.experiment_service:RestoreCheckpoint: Restoring checkpoint from hash: 802319f761e1ba3dc3472a78 (weights-only, target_step=24)\n", + "16/07/2026-08:40:41.096 INFO:weightslab.trainer.services.experiment_service:RestoreCheckpoint: Pausing training before restore...\n", + "16/07/2026-08:40:41.099 INFO:weightslab.components.global_monitoring:pause: \n", + "Training paused.\n", + "16/07/2026-08:40:41.102 INFO:weightslab.components.checkpoint_manager:load_state: \n", + "============================================================\n", + "16/07/2026-08:40:41.103 INFO:weightslab.components.checkpoint_manager:load_state: Loading and applying state: 802319f761e1ba3d...\n", + "16/07/2026-08:40:41.104 INFO:weightslab.components.checkpoint_manager:load_state: ============================================================\n", + "16/07/2026-08:40:41.110 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loading checkpoint 802319f761e1ba3d...\n", + "16/07/2026-08:40:41.112 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Target: HP=802319f7 MODEL=61e1ba3d DATA=c3472a78\n", + "16/07/2026-08:40:41.115 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Current: HP=bfb454d6 MODEL=d39e7bd9 DATA=43e44972\n", + "16/07/2026-08:40:41.118 WARNING:weightslab.components.checkpoint_manager:load_checkpoint: [WARNING] Model architecture file not found: /tmp/tmpf35sd4_z/checkpoints/models/61e1ba3d/61e1ba3d_architecture.pkl\n", + "16/07/2026-08:40:41.176 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [OK] Loaded weights from step 25 with RNG state\n", + "16/07/2026-08:40:41.184 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [OK] Loaded config (hash changed)\n", + "16/07/2026-08:40:41.198 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [OK] Loaded data snapshot (170 rows) with RNG state\n", + "16/07/2026-08:40:41.200 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loaded components: {'data', 'weights', 'config'}\n", + "16/07/2026-08:40:41.215 INFO:weightslab.components.checkpoint_manager:load_state: [OK] Applied weights to existing model (step 25)\n", + "16/07/2026-08:40:41.217 INFO:weightslab.components.checkpoint_manager:load_state: Loaded optimizer state\n", + "16/07/2026-08:40:41.219 INFO:weightslab.components.checkpoint_manager:load_state: [OK] Applied hyperparameters config\n", + "16/07/2026-08:40:41.254 INFO:weightslab.components.checkpoint_manager:load_state: [OK] Applied data snapshot (170 rows)\n", + "16/07/2026-08:40:41.266 INFO:weightslab.components.checkpoint_manager:load_state: \n", + "[OK] Successfully loaded and applied state: 802319f761e1ba3d\n", + "16/07/2026-08:40:41.267 INFO:weightslab.components.checkpoint_manager:load_state: ============================================================\n", + "\n", + "16/07/2026-08:40:41.272 INFO:weightslab.trainer.services.experiment_service:RestoreCheckpoint: Successfully restored checkpoint: 802319f761e1ba3dc3472a78\n", + "16/07/2026-08:40:44.091 INFO:weightslab.components.global_monitoring:pause: \n", + "Training paused.\n", + "16/07/2026-08:40:44.092 INFO:weightslab.trainer.services.experiment_service:ExperimentCommand: [WeightsLab] UI Command: HP changed, experiment paused!\n", + "16/07/2026-08:40:44.093 INFO:weightslab.trainer.services.experiment_service:ExperimentCommand: \n", + "[WeightsLab] UI Command: RESUME\n", + "16/07/2026-08:40:44.096 INFO:weightslab.components.global_monitoring:resume: \n", + "Attempting to resume training...\n", + "16/07/2026-08:40:44.104 INFO:weightslab.components.experiment_hash:has_changed: Experiment configuration changed: {'model'}\n", + "16/07/2026-08:40:44.107 INFO:weightslab.components.experiment_hash:generate_hash: Generated experiment hash: 802319f7f5ee131bc3472a78- (HP: 802319f7, Model: f5ee131b, Data: c3472a78)\n", + "16/07/2026-08:40:44.111 INFO:weightslab.components.checkpoint_manager:update_experiment_hash: New experiment hash: 802319f7-f5ee131b-c3472a78 (previous: 802319f7-61e1ba3d-c3472a78)\n", + "16/07/2026-08:40:44.113 INFO:weightslab.components.checkpoint_manager:update_experiment_hash: Changed components: {'model'}\n", + "16/07/2026-08:40:44.115 INFO:weightslab.components.checkpoint_manager:update_experiment_hash: Changes pending (not dumped yet): {'model'}\n", + "16/07/2026-08:40:44.118 INFO:weightslab.components.checkpoint_manager:save_pending_changes: Dumping pending changes: {'model'}\n", + "16/07/2026-08:40:44.427 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmpf35sd4_z/checkpoints/loggers/loggers.manifest.json (1 chunks)\n", + "16/07/2026-08:40:44.434 INFO:weightslab.components.global_monitoring:resume: Hashes by module: ['802319f7', 'f5ee131b', 'c3472a78']\n", + "16/07/2026-08:40:44.437 INFO:weightslab.components.global_monitoring:resume: Resuming training now...\n", + "16/07/2026-08:40:44.438 INFO:weightslab.components.global_monitoring:resume: Hashes by module on resume: ['802319f7', 'f5ee131b', 'c3472a78']\n", + "16/07/2026-08:40:44.448 INFO:weightslab.backend.dataloader_interface:set_batch_size: Batch size updated: 16 -> 8 (Loader: train_loader)\n", + "16/07/2026-08:40:44.452 INFO:weightslab.components.global_monitoring:resume: \n", + "Training resumed as modules hashes have been computed: ['802319f7', 'f5ee131b', 'c3472a78'].\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n", + "\n", + "Training: 22%|██▏ | 327/1500 [07:07<23:56, 1.22s/it, iou=54.03%, test_loss=7.9861, train_loss=10.4120]\u001b[A\n", + "Training: 22%|██▏ | 328/1500 [07:07<48:08, 2.46s/it, iou=54.03%, test_loss=7.9861, train_loss=10.4120]\u001b[A\n", + "Training: 22%|██▏ | 328/1500 [07:08<48:08, 2.46s/it, iou=54.03%, test_loss=7.9861, train_loss=8.1298] \u001b[A\n", + "Training: 22%|██▏ | 329/1500 [07:08<36:27, 1.87s/it, iou=54.03%, test_loss=7.9861, train_loss=8.1298]\u001b[A\n", + "Training: 22%|██▏ | 329/1500 [07:08<36:27, 1.87s/it, iou=54.03%, test_loss=7.9861, train_loss=8.6498]\u001b[A\n", + "Training: 22%|██▏ | 330/1500 [07:08<27:35, 1.41s/it, iou=54.03%, test_loss=7.9861, train_loss=8.6498]\u001b[A\n", + "Training: 22%|██▏ | 330/1500 [07:09<27:35, 1.41s/it, iou=54.03%, test_loss=7.9861, train_loss=8.4801]\u001b[A\n", + "Training: 22%|██▏ | 331/1500 [07:09<22:37, 1.16s/it, iou=54.03%, test_loss=7.9861, train_loss=8.4801]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:40:48.449 INFO:weightslab.components.checkpoint_manager:save_model_checkpoint: Saved model checkpoint: 802319f7f5ee131bc3472a78_step_000030.pt\n", + "16/07/2026-08:40:48.825 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmpf35sd4_z/checkpoints/loggers/loggers.manifest.json (1 chunks)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 22%|██▏ | 331/1500 [07:11<22:37, 1.16s/it, iou=54.03%, test_loss=7.9861, train_loss=9.8237]\u001b[A\n", + "Training: 22%|██▏ | 332/1500 [07:11<31:52, 1.64s/it, iou=54.03%, test_loss=7.9861, train_loss=9.8237]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:40:51.421 INFO:weightslab.backend.dataloader_interface:set_batch_size: Batch size updated: 16 -> 8 (Loader: test_loader)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:40:56.551 WARNING:weightslab.watchdog.grpc_watchdog:unary_unary_wrapper: [gRPC] /ExperimentService/GetDataSamples completed in 3558.8ms\n", + "16/07/2026-08:41:00.986 INFO:weightslab.src:write_dataframe: write_dataframe: wrote 593 row(s) × 24 column(s) as json to /tmp/tmpf35sd4_z/d8e01c87_dataframe.json\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 22%|██▏ | 332/1500 [07:23<31:52, 1.64s/it, iou=90.43%, test_loss=15.9695, train_loss=9.6350]\u001b[A\n", + "Training: 22%|██▏ | 333/1500 [07:23<1:32:06, 4.74s/it, iou=90.43%, test_loss=15.9695, train_loss=9.6350]\u001b[A\n", + "Training: 22%|██▏ | 333/1500 [07:24<1:32:06, 4.74s/it, iou=90.43%, test_loss=15.9695, train_loss=7.9581]\u001b[A\n", + "Training: 22%|██▏ | 334/1500 [07:24<1:06:21, 3.41s/it, iou=90.43%, test_loss=15.9695, train_loss=7.9581]\u001b[A\n", + "Training: 22%|██▏ | 334/1500 [07:24<1:06:21, 3.41s/it, iou=90.43%, test_loss=15.9695, train_loss=7.3944]\u001b[A\n", + "Training: 22%|██▏ | 335/1500 [07:24<47:55, 2.47s/it, iou=90.43%, test_loss=15.9695, train_loss=7.3944] \u001b[A\n", + "Training: 22%|██▏ | 335/1500 [07:24<47:55, 2.47s/it, iou=90.43%, test_loss=15.9695, train_loss=10.4550]\u001b[A\n", + "Training: 22%|██▏ | 336/1500 [07:24<35:17, 1.82s/it, iou=90.43%, test_loss=15.9695, train_loss=10.4550]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:41:02.160 INFO:weightslab.components.checkpoint_manager:save_model_checkpoint: Saved model checkpoint: 802319f7f5ee131bc3472a78_step_000035.pt\n", + "16/07/2026-08:41:02.346 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmpf35sd4_z/checkpoints/loggers/loggers.manifest.json (1 chunks)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 22%|██▏ | 336/1500 [07:25<35:17, 1.82s/it, iou=90.43%, test_loss=15.9695, train_loss=7.6845] \u001b[A\n", + "Training: 22%|██▏ | 337/1500 [07:25<29:14, 1.51s/it, iou=90.43%, test_loss=15.9695, train_loss=7.6845]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:41:05.682 INFO:weightslab.src:write_dataframe: write_dataframe: wrote 593 row(s) × 24 column(s) as json to /tmp/tmpf35sd4_z/d8e01c87_dataframe.json\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 22%|██▏ | 337/1500 [07:28<29:14, 1.51s/it, iou=55.45%, test_loss=9.3424, train_loss=6.8015] \u001b[A\n", + "Training: 23%|██▎ | 338/1500 [07:28<38:06, 1.97s/it, iou=55.45%, test_loss=9.3424, train_loss=6.8015]\u001b[A\n", + "Training: 23%|██▎ | 338/1500 [07:28<38:06, 1.97s/it, iou=55.45%, test_loss=9.3424, train_loss=8.4716]\u001b[A\n", + "Training: 23%|██▎ | 339/1500 [07:28<28:11, 1.46s/it, iou=55.45%, test_loss=9.3424, train_loss=8.4716]\u001b[A\n", + "Training: 23%|██▎ | 339/1500 [07:29<28:11, 1.46s/it, iou=55.45%, test_loss=9.3424, train_loss=8.7423]\u001b[A\n", + "Training: 23%|██▎ | 340/1500 [07:29<21:16, 1.10s/it, iou=55.45%, test_loss=9.3424, train_loss=8.7423]\u001b[A\n", + "Training: 23%|██▎ | 340/1500 [07:29<21:16, 1.10s/it, iou=55.45%, test_loss=9.3424, train_loss=8.9636]\u001b[A\n", + "Training: 23%|██▎ | 341/1500 [07:29<16:33, 1.17it/s, iou=55.45%, test_loss=9.3424, train_loss=8.9636]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:41:06.887 INFO:weightslab.components.checkpoint_manager:save_model_checkpoint: Saved model checkpoint: 802319f7f5ee131bc3472a78_step_000040.pt\n", + "16/07/2026-08:41:07.490 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmpf35sd4_z/checkpoints/loggers/loggers.manifest.json (1 chunks)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 23%|██▎ | 341/1500 [07:30<16:33, 1.17it/s, iou=55.45%, test_loss=9.3424, train_loss=8.2650]\u001b[A\n", + "Training: 23%|██▎ | 342/1500 [07:30<17:36, 1.10it/s, iou=55.45%, test_loss=9.3424, train_loss=8.2650]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:41:10.072 INFO:weightslab.src:write_dataframe: write_dataframe: wrote 593 row(s) × 24 column(s) as json to /tmp/tmpf35sd4_z/d8e01c87_dataframe.json\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 23%|██▎ | 342/1500 [07:32<17:36, 1.10it/s, iou=57.76%, test_loss=8.9165, train_loss=9.8330]\u001b[A\n", + "Training: 23%|██▎ | 343/1500 [07:32<26:56, 1.40s/it, iou=57.76%, test_loss=8.9165, train_loss=9.8330]\u001b[A\n", + "Training: 23%|██▎ | 343/1500 [07:33<26:56, 1.40s/it, iou=57.76%, test_loss=8.9165, train_loss=8.5029]\u001b[A\n", + "Training: 23%|██▎ | 344/1500 [07:33<20:40, 1.07s/it, iou=57.76%, test_loss=8.9165, train_loss=8.5029]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n", + "\n", + "Training: 23%|██▎ | 344/1500 [07:33<20:40, 1.07s/it, iou=57.76%, test_loss=8.9165, train_loss=6.6450]\u001b[A\n", + "Training: 23%|██▎ | 345/1500 [07:33<16:02, 1.20it/s, iou=57.76%, test_loss=8.9165, train_loss=6.6450]\u001b[A\n", + "Training: 23%|██▎ | 345/1500 [07:33<16:02, 1.20it/s, iou=57.76%, test_loss=8.9165, train_loss=7.5255]\u001b[A\n", + "Training: 23%|██▎ | 346/1500 [07:33<13:05, 1.47it/s, iou=57.76%, test_loss=8.9165, train_loss=7.5255]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:41:11.459 INFO:weightslab.components.checkpoint_manager:save_model_checkpoint: Saved model checkpoint: 802319f7f5ee131bc3472a78_step_000045.pt\n", + "16/07/2026-08:41:11.769 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmpf35sd4_z/checkpoints/loggers/loggers.manifest.json (1 chunks)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 23%|██▎ | 346/1500 [07:34<13:05, 1.47it/s, iou=57.76%, test_loss=8.9165, train_loss=7.0646]\u001b[A\n", + "Training: 23%|██▎ | 347/1500 [07:34<15:19, 1.25it/s, iou=57.76%, test_loss=8.9165, train_loss=7.0646]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:41:16.032 INFO:weightslab.src:write_dataframe: write_dataframe: wrote 593 row(s) × 24 column(s) as json to /tmp/tmpf35sd4_z/d8e01c87_dataframe.json\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 23%|██▎ | 347/1500 [07:38<15:19, 1.25it/s, iou=58.16%, test_loss=8.7419, train_loss=8.9887]\u001b[A\n", + "Training: 23%|██▎ | 348/1500 [07:38<33:36, 1.75s/it, iou=58.16%, test_loss=8.7419, train_loss=8.9887]\u001b[A\n", + "Training: 23%|██▎ | 348/1500 [07:39<33:36, 1.75s/it, iou=58.16%, test_loss=8.7419, train_loss=7.1660]\u001b[A\n", + "Training: 23%|██▎ | 349/1500 [07:39<25:01, 1.30s/it, iou=58.16%, test_loss=8.7419, train_loss=7.1660]\u001b[A\n", + "Training: 23%|██▎ | 349/1500 [07:39<25:01, 1.30s/it, iou=58.16%, test_loss=8.7419, train_loss=7.0642]\u001b[A\n", + "Training: 23%|██▎ | 350/1500 [07:39<18:58, 1.01it/s, iou=58.16%, test_loss=8.7419, train_loss=7.0642]\u001b[A\n", + "Training: 23%|██▎ | 350/1500 [07:39<18:58, 1.01it/s, iou=58.16%, test_loss=8.7419, train_loss=5.4376]\u001b[A\n", + "Training: 23%|██▎ | 351/1500 [07:39<14:41, 1.30it/s, iou=58.16%, test_loss=8.7419, train_loss=5.4376]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:41:17.068 INFO:weightslab.components.checkpoint_manager:save_model_checkpoint: Saved model checkpoint: 802319f7f5ee131bc3472a78_step_000050.pt\n", + "16/07/2026-08:41:17.255 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmpf35sd4_z/checkpoints/loggers/loggers.manifest.json (1 chunks)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 23%|██▎ | 351/1500 [07:40<14:41, 1.30it/s, iou=58.16%, test_loss=8.7419, train_loss=8.3300]\u001b[A\n", + "Training: 23%|██▎ | 352/1500 [07:40<14:17, 1.34it/s, iou=58.16%, test_loss=8.7419, train_loss=8.3300]\u001b[A/usr/local/lib/python3.12/dist-packages/torch/utils/data/dataloader.py:668: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used.\n", + " warnings.warn(warn_msg)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:41:20.421 INFO:weightslab.src:write_dataframe: write_dataframe: wrote 593 row(s) × 24 column(s) as json to /tmp/tmpf35sd4_z/d8e01c87_dataframe.json\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 23%|██▎ | 352/1500 [07:43<14:17, 1.34it/s, iou=58.39%, test_loss=8.7055, train_loss=6.9667]\u001b[A\n", + "Training: 24%|██▎ | 353/1500 [07:43<26:44, 1.40s/it, iou=58.39%, test_loss=8.7055, train_loss=6.9667]\u001b[A\n", + "Training: 24%|██▎ | 353/1500 [07:43<26:44, 1.40s/it, iou=58.39%, test_loss=8.7055, train_loss=5.6257]\u001b[A\n", + "Training: 24%|██▎ | 354/1500 [07:43<20:19, 1.06s/it, iou=58.39%, test_loss=8.7055, train_loss=5.6257]\u001b[A\n", + "Training: 24%|██▎ | 354/1500 [07:43<20:19, 1.06s/it, iou=58.39%, test_loss=8.7055, train_loss=6.9397]\u001b[A\n", + "Training: 24%|██▎ | 355/1500 [07:43<15:46, 1.21it/s, iou=58.39%, test_loss=8.7055, train_loss=6.9397]\u001b[A\n", + "Training: 24%|██▎ | 355/1500 [07:44<15:46, 1.21it/s, iou=58.39%, test_loss=8.7055, train_loss=7.1436]\u001b[A\n", + "Training: 24%|██▎ | 356/1500 [07:44<12:38, 1.51it/s, iou=58.39%, test_loss=8.7055, train_loss=7.1436]\u001b[A" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "16/07/2026-08:41:21.337 INFO:weightslab.components.global_monitoring:pause: \n", + "Training paused.\n", + "16/07/2026-08:41:21.339 INFO:weightslab.trainer.services.experiment_service:ExperimentCommand: [WeightsLab] UI Command: HP changed, experiment paused!\n", + "16/07/2026-08:41:21.340 INFO:weightslab.trainer.services.experiment_service:ExperimentCommand: \n", + "[WeightsLab] UI Command: PAUSE\n", + "16/07/2026-08:41:21.342 INFO:weightslab.components.global_monitoring:pause: \n", + "Training paused.\n", + "16/07/2026-08:41:21.615 INFO:weightslab.components.checkpoint_manager:save_model_checkpoint: Saved model checkpoint: 802319f7f5ee131bc3472a78_step_000055.pt\n", + "16/07/2026-08:41:21.811 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmpf35sd4_z/checkpoints/loggers/loggers.manifest.json (1 chunks)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\n", + "Training: 24%|██▎ | 356/1500 [07:45<12:38, 1.51it/s, iou=58.39%, test_loss=8.7055, train_loss=6.4582]\u001b[A\n", + "Training: 24%|██▍ | 357/1500 [07:45<14:34, 1.31it/s, iou=58.39%, test_loss=8.7055, train_loss=6.4582]\u001b[A" + ] + } + ], + "source": [ + "# Start the background gRPC server + a public bore tunnel so a local Weights Studio can attach.\n", + "wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "# Flip the experiment into the training state, then run the loop.\n", + "wl.start_training(timeout=3)\n", + "\n", + "# training_steps_to_do is None (open-ended) in the example; cap the demo to a finite\n", + "# number of steps so the cell terminates on its own on Colab (raise for a longer run).\n", + "max_steps = config[\"training_steps_to_do\"] or 1500\n", + "eval_ratio = config[\"eval_full_to_train_steps_ratio\"]\n", + "export_ratio = config[\"experiment_dump_to_train_steps_ratio\"]\n", + "\n", + "test_loss, test_metric = None, None\n", + "pbar = tqdm.tqdm(range(max_steps), desc=\"Training\")\n", + "for train_step in pbar:\n", + " age = model.get_age() if hasattr(model, \"get_age\") else train_step\n", + "\n", + " # Train one step.\n", + " train_loss = train(train_loader, model, optimizer, train_sig, device, grid_size, conf_thresh)\n", + "\n", + " # Full eval pass every eval_ratio steps.\n", + " if age == 0 or age % eval_ratio == 0:\n", + " test_loader_len = len(test_loader)\n", + " test_loss, test_metric = test(test_loader, model, test_sig, device, grid_size, conf_thresh, test_loader_len)\n", + "\n", + " # Export per-sample history + dataframe periodically.\n", + " if age > 0 and age % export_ratio == 0:\n", + " wl.write_history()\n", + " wl.write_dataframe()\n", + "\n", + " pbar.set_postfix(\n", + " train_loss=f\"{train_loss:.4f}\",\n", + " test_loss=f\"{test_loss:.4f}\" if test_loss is not None else \"N/A\",\n", + " iou=f\"{test_metric:.2f}%\" if test_metric is not None else \"N/A\",\n", + " )\n", + "\n", + "wl.write_history()\n", + "wl.write_dataframe()\n", + "print(\"Training complete. Logs at:\", log_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "j9TYziOc_xBg" + }, + "source": [ + "## See it live in Weights Studio\n", + "\n", + "**Colab has no Docker daemon**, so run Studio on your own machine and point it at this notebook's\n", + "backend using the `bore.pub:` printed in the Expose section:\n", + "\n", + "```bash\n", + "pip install weightslab\n", + "weightslab start # Terminal 1 - opens http://localhost:5173\n", + "weightslab tunnel bore.pub:12345 # Terminal 2 - the host:port from the Expose section\n", + "```\n", + "\n", + "Then open **http://localhost:5173**. Prefer local-only? Run this example directly on your machine\n", + "(`weightslab start example` selects a bundled example) and launch the UI next to it - no tunnel." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oQsjwjg7_xBg" + }, + "source": [ + "---\n", + "\n", + "
\n", + "Crafted by GrayboxTech - if WeightsLab helps you catch a bad label, drop us a star on GitHub.\n", + "
" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "wl-detection.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/PyTorch/wl-fraud-detection.ipynb b/weightslab/examples/Notebooks/PyTorch/wl-fraud-detection.ipynb new file mode 100644 index 00000000..ea8b274f --- /dev/null +++ b/weightslab/examples/Notebooks/PyTorch/wl-fraud-detection.ipynb @@ -0,0 +1,153 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "\"Open" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "
\n\n \n \"WeightsLab\n\n \"License\"\n \"Stars\"\n \"Version\"\n
\n \"Open\n\n Welcome to the WeightsLab image-classification notebook! WeightsLab is an open-source PyTorch tool for dataset debugging, mislabel detection, and mid-training data curation. Browse the Docs for details, and raise an issue on GitHub for support.
" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "# Bank Fraud Detection with WeightsLab (tabular)\n\nThis notebook trains a small MLP to flag **fraudulent bank card transactions** and instruments it with WeightsLab so every training signal is traced **back to the exact transaction** producing it.\n\nThere are no images: a sample is one transaction (a **row**), the model input **is** the 16-feature vector, and WeightsLab sends that vector to the UI as a `vector` (not a fake image). Each raw feature (`amount`, `merchant_risk`, `geo_distance_km`, \u2026) is a **sortable column** in the List Exploration view.\n\n### What you'll do\n1. Install WeightsLab.\n2. Generate a reproducible synthetic fraud stream (no download).\n3. Wrap the model, optimizer, dataloaders, loss and metric.\n4. Train while streaming per-transaction loss/prediction to Weights Studio.\n\n*Real drop-in datasets: Kaggle Credit Card Fraud (ULB), PaySim.*" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Setup\n\nInstall WeightsLab from PyPI. These tabular demos are tiny \u2014 the free Colab CPU runtime is plenty (no GPU needed).\n\n> The **tabular input path** (the feature vector is sent to the UI as a `vector` \u2014 not a fake image) needs a WeightsLab build with tabular support (the `252-tabular-experiment` line or a release that includes it)." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "\"PyPI\n\"PyPI\n\"PyPI" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "%pip install torch torchvision" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "%pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev6\"" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## 1. Imports\n\n`weightslab` is imported as `wl`. The two `guard_*_context` managers scope a block as training vs. evaluation so signals are attributed to the right phase." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import tempfile, logging\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset\nfrom torchmetrics.classification import Accuracy\nfrom tqdm.auto import tqdm\n\nimport weightslab as wl\nfrom weightslab.components.global_monitoring import (\n guard_training_context,\n guard_testing_context,\n)\n\nlogging.basicConfig(level=logging.ERROR)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## 2. A synthetic fraud dataset (rows, not images)\n\n`make_synthetic_fraud` builds reproducible transactions: 16 numeric features with a binary label (~12% fraud). Fraud rows come from shifted distributions (larger amounts, off-hours activity, higher merchant risk, device changes, larger geo jumps). The dataset returns the **standardized feature vector** as the model input, and `get_items(...)` exposes the **raw** values as metadata columns." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "FEATURE_NAMES = [\n \"amount\", \"old_balance\", \"new_balance\", \"balance_delta\",\n \"txn_hour\", \"txn_day_of_week\", \"txn_count_1h\", \"txn_count_24h\",\n \"avg_amount_7d\", \"std_amount_7d\", \"merchant_risk\", \"device_change\",\n \"geo_distance_km\", \"is_foreign\", \"account_age_days\", \"num_prior_disputes\",\n]\nNUM_FEATURES = len(FEATURE_NAMES) # 16\nFRAUD_RATE = 0.12\n\n\ndef make_synthetic_fraud(n_samples, seed=0):\n \"\"\"Return (X_std, X_raw, y): standardized input, raw display values, label.\"\"\"\n rng = np.random.default_rng(seed)\n n_fraud = max(1, round(n_samples * FRAUD_RATE))\n n_legit = max(1, n_samples - n_fraud)\n\n def draw(n, fraud):\n amount = rng.gamma(2.0, 180.0 if fraud else 60.0, n)\n old_balance = rng.gamma(2.0, 800.0, n)\n spent = amount * (rng.uniform(0.6, 1.4, n) if fraud else rng.uniform(0.0, 0.6, n))\n new_balance = np.clip(old_balance - spent, 0, None)\n balance_delta = old_balance - new_balance\n txn_hour = (rng.normal(2.5, 2.0, n) % 24) if fraud else rng.normal(13.0, 4.0, n)\n txn_dow = rng.integers(0, 7, n).astype(float)\n txn_count_1h = rng.poisson(4.0 if fraud else 1.0, n).astype(float)\n txn_count_24h = rng.poisson(18.0 if fraud else 6.0, n).astype(float)\n avg_amount_7d = rng.gamma(2.0, 90.0 if fraud else 70.0, n)\n std_amount_7d = rng.gamma(2.0, 60.0 if fraud else 25.0, n)\n merchant_risk = rng.beta(5.0, 2.0, n) if fraud else rng.beta(2.0, 6.0, n)\n device_change = rng.binomial(1, 0.55 if fraud else 0.08, n).astype(float)\n geo = rng.gamma(2.0, 400.0 if fraud else 25.0, n)\n is_foreign = rng.binomial(1, 0.45 if fraud else 0.05, n).astype(float)\n account_age = rng.gamma(2.0, 120.0 if fraud else 500.0, n)\n disputes = rng.poisson(1.5 if fraud else 0.2, n).astype(float)\n return np.stack([amount, old_balance, new_balance, balance_delta, txn_hour,\n txn_dow, txn_count_1h, txn_count_24h, avg_amount_7d,\n std_amount_7d, merchant_risk, device_change, geo,\n is_foreign, account_age, disputes], axis=1)\n\n x_raw = np.concatenate([draw(n_legit, False), draw(n_fraud, True)]).astype(np.float64)\n y = np.concatenate([np.zeros(n_legit, np.int64), np.ones(n_fraud, np.int64)])\n mean, std = x_raw.mean(0, keepdims=True), x_raw.std(0, keepdims=True)\n std[std == 0] = 1.0\n x_std = (x_raw - mean) / std\n perm = rng.permutation(len(x_raw))\n return x_std[perm].astype(np.float32), x_raw[perm].astype(np.float32), y[perm]\n\n\nclass FraudDataset(Dataset):\n \"\"\"Yields (feature_vector, id, label); get_items() adds feature columns.\"\"\"\n\n def __init__(self, n_samples, seed=0):\n xs, xr, y = make_synthetic_fraud(n_samples, seed)\n self.features = torch.from_numpy(xs) # model input\n self.raw = xr # display values\n self.labels = torch.from_numpy(y)\n\n def __len__(self):\n return len(self.features)\n\n def __getitem__(self, idx):\n return self.features[idx], idx, int(self.labels[idx])\n\n def get_items(self, idx, include_metadata=False, include_labels=False, include_images=False):\n x = self.features[idx] if include_images else None\n target = int(self.labels[idx]) if include_labels else None\n meta = ({n: round(float(self.raw[idx][i]), 4) for i, n in enumerate(FEATURE_NAMES)}\n if include_metadata else None)\n return x, idx, target, meta\n\n\nclass FraudMLP(nn.Module):\n def __init__(self, in_features=NUM_FEATURES, hidden=64, num_classes=2):\n super().__init__()\n self.net = nn.Sequential(\n nn.Flatten(), nn.Linear(in_features, hidden), nn.ReLU(), nn.Dropout(0.1),\n nn.Linear(hidden, hidden // 2), nn.ReLU(), nn.Linear(hidden // 2, num_classes))\n\n def forward(self, x):\n return self.net(x)\n\n\ndef build_dataset(n, seed=0):\n return FraudDataset(n, seed)\n\n\ndef build_model():\n return FraudMLP()\n" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## 3. Configuration\n\nEvery tunable lives here in one dict, like a `config.yaml` with comments. Wrapping it with `flag=\"hyperparameters\"` lets Weights Studio read (and live-edit) these values while training." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "config = {\n \"experiment_name\": \"fraud_detection_mlp\",\n \"device\": str(device),\n \"root_log_dir\": tempfile.mkdtemp(prefix=\"weightslab_fraud_\"),\n \"learning_rate\": 0.005,\n \"training_steps_to_do\": 2000,\n \"eval_full_to_train_steps_ratio\": 100,\n \"write_export_ratio\": 500,\n \"class_weights\": [1.0, 4.0], # [legit, fraud] \u2014 up-weight fraud\n \"dataset\": {\"seed\": 0, \"n_train\": 4000, \"n_test\": 1000},\n \"data\": {\"train_loader\": {\"batch_size\": 32},\n \"test_loader\": {\"batch_size\": 128}},\n}\nwl.watch_or_edit(config, flag=\"hyperparameters\", poll_interval=1.0)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## 4. Wrap the training objects\n\nThis is the heart of WeightsLab. Each object passes through `wl.watch_or_edit(...)` with a `flag` describing its role. The tracked dataset's `get_items()` exposes every feature as a **sortable column**; `preload_metadata=True` loads them at init." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "cfg = config\ntrain_ds = build_dataset(cfg['dataset']['n_train'], seed=cfg['dataset']['seed'])\ntest_ds = build_dataset(cfg['dataset']['n_test'], seed=cfg['dataset']['seed'] + 1)\n\nmodel = wl.watch_or_edit(build_model().to(device), flag='model', device=device)\noptimizer = wl.watch_or_edit(\n optim.Adam(model.parameters(), lr=cfg['learning_rate']), flag='optimizer')\n\ntrain_loader = wl.watch_or_edit(\n train_ds, flag='data', loader_name='train_loader',\n batch_size=cfg['data']['train_loader']['batch_size'], shuffle=True,\n is_training=True, preload_labels=True, preload_metadata=True)\ntest_loader = wl.watch_or_edit(\n test_ds, flag='data', loader_name='test_loader',\n batch_size=cfg['data']['test_loader']['batch_size'], shuffle=False,\n is_training=False, preload_labels=True, preload_metadata=True)\n\n# Class weights counter the minority (positive) class prevalence.\ncw = torch.tensor(cfg['class_weights'], dtype=torch.float32, device=device)\ntrain_criterion = wl.watch_or_edit(\n nn.CrossEntropyLoss(weight=cw, reduction='none'),\n flag='loss', signal_name='train-loss-CE', log=True)\ntest_criterion = wl.watch_or_edit(\n nn.CrossEntropyLoss(weight=cw, reduction='none'),\n flag='loss', signal_name='test-loss-CE', log=True)\nmetric = wl.watch_or_edit(\n Accuracy(task='multiclass', num_classes=2).to(device),\n flag='metric', signal_name='metric-ACC', log=True)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## 5. Train and evaluate steps\n\nThe `guard_training_context` / `guard_testing_context` blocks tell WeightsLab the phase. `criterion(..., batch_ids=ids, preds=preds)` passes the sample ids so loss is stored **per sample**, and `wl.save_signals(...)` logs a custom per-sample accuracy signal during evaluation." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "def train(loader, model, optimizer, criterion, device):\n with guard_training_context:\n inputs, ids, labels = next(loader)\n inputs, labels = inputs.to(device), labels.to(device)\n optimizer.zero_grad()\n logits = model(inputs)\n preds = logits.argmax(dim=1, keepdim=True)\n loss = criterion(logits.float(), labels.long(), batch_ids=ids, preds=preds)\n total = loss.mean()\n total.backward()\n optimizer.step()\n return total.detach().cpu().item()\n\n\ndef test(loader, model, criterion, metric, device, n_batches):\n losses = torch.tensor(0.0, device=device)\n for inputs, ids, labels in loader:\n with guard_testing_context:\n inputs, labels = inputs.to(device), labels.to(device)\n logits = model(inputs)\n preds = logits.argmax(dim=1, keepdim=True)\n losses += criterion(logits, labels, batch_ids=ids, preds=preds).mean()\n metric.update(logits, labels)\n correct = (preds.view(-1) == labels.view(-1)).float()\n wl.save_signals(preds_raw=logits, targets=labels, batch_ids=ids,\n signals={\"test_metric/Accuracy_per_sample\": correct}, preds=preds)\n return (losses / max(1, n_batches)).item(), (metric.compute() * 100).item()\n" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## 6. Serve and train\n\n`wl.serve(serving_grpc=True, serving_bore=True)` starts the background gRPC server (non-blocking) and a `bore.pub` tunnel so Weights Studio on your own machine can reach this Colab backend. `wl.start_training(...)` flips the experiment into the *training* state, then we run the loop." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "wl.serve(serving_grpc=True, serving_bore=True)" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "wl.start_training(timeout=3)\n\nsteps = config['training_steps_to_do']\neval_ratio = config['eval_full_to_train_steps_ratio']\nn_test_batches = len(test_loader)\n\ntest_loss = test_acc = None\npbar = tqdm(range(steps), desc='Training')\nfor step in pbar:\n age = model.get_age() if hasattr(model, 'get_age') else step\n train_loss = train(train_loader, model, optimizer, train_criterion, device)\n if age > 0 and age % eval_ratio == 0:\n test_loss, test_acc = test(test_loader, model, test_criterion, metric, device, n_test_batches)\n postfix = {'loss': f'{train_loss:.3f}'}\n if test_acc is not None:\n postfix['test_acc'] = f'{test_acc:.1f}%'\n pbar.set_postfix(postfix)\n\nwl.write_history()\nwl.write_dataframe()\nprint('Training complete.')" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## See it live in Weights Studio\n\nEverything above runs headless. The payoff is **Weights Studio**, where each row is one record and every feature is a **sortable column**.\n\nStudio runs as a local Docker stack, and **Colab has no Docker daemon**, so you run Studio on your own machine and point it at this notebook's backend via the `bore.pub:` endpoint **printed in Section 6**.\n\n**On your machine** (with Docker Desktop):\n```bash\npip install weightslab\nweightslab start # opens http://localhost:5173\nweightslab tunnel bore.pub:12345 # in another window, the host:port printed in Section 6\n```\n\nThen open **http://localhost:5173** and switch the Data Exploration board to the **List** view." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Curate in the UI\n\nIn Weights Studio, switch the Data Exploration board to the **List** view:\n\n1. **Sort by `train-loss-CE` descending** and generate its histogram to find the hardest transactions \u2014 usually the frauds the model still misses.\n2. **Lock** the loss sort, then add `merchant_risk` or `amount` as a secondary sort to see which feature ranges drive the errors.\n3. **Right-click a column** to clone it, reset the sort, or generate a histogram \u2014 the same actions you use on image datasets.\n4. **Discard** mislabeled or leaked rows and keep training \u2014 no restart." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "---\n\n
\nCrafted by GrayboxTech - if WeightsLab helps you catch a bad label, drop us a star on GitHub.\n
" + } + ], + "metadata": { + "colab": { + "name": "wl-fraud-detection.ipynb", + "provenance": [], + "include_colab_link": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/PyTorch/wl-segmentation.ipynb b/weightslab/examples/Notebooks/PyTorch/wl-segmentation.ipynb new file mode 100644 index 00000000..dd7490f3 --- /dev/null +++ b/weightslab/examples/Notebooks/PyTorch/wl-segmentation.ipynb @@ -0,0 +1,1072 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "06579270", + "metadata": { + "id": "06579270" + }, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Welcome to the WeightsLab Semantic Segmentation (BDD100k) notebook! WeightsLab traces training signals back to the exact samples producing them. Browse the Docs for details.
" + ] + }, + { + "cell_type": "markdown", + "id": "c122b485", + "metadata": { + "id": "c122b485" + }, + "source": [ + "# Semantic Segmentation (BDD100k) with WeightsLab\n", + "\n", + "Trains a small U-Net on a **BDD100k**-style dataset and instruments per-**sample** Dice (metric) and CrossEntropy (loss), so every mask's quality is traced back to the exact sample producing it.\n", + "\n", + "### What you'll do\n", + "1. Install WeightsLab.\n", + "2. Download your dataset **zip from a Google Drive link** and unzip it (the training **code is inlined** below).\n", + "3. Set every knob in one **config** dict (like a `config.yaml`).\n", + "4. Wrap the dataloaders, model, optimizer, and per-sample Dice + CrossEntropy signals with the SDK.\n", + "5. Train while per-sample signals are captured, then (optionally) open the live **Weights Studio** UI.\n", + "\n", + "> Data note: this notebook fetches your dataset from a **Drive share link** you provide (`DATASET_URL`); all training code (dataset, model, criterions, loops) is **inlined below** - no `main.py`, no full-repo clone. The zip should unpack to a BDD100k-style layout (`images/{train,val}/` + `labels/{train,val}/`), one semantic mask per image." + ] + }, + { + "cell_type": "markdown", + "id": "ec299273", + "metadata": { + "id": "ec299273" + }, + "source": [ + "## Setup\n", + "\n", + "On Colab, pick a **T4 GPU** runtime (`Runtime -> Change runtime type -> T4 GPU`)." + ] + }, + { + "cell_type": "markdown", + "id": "db7e968a", + "metadata": { + "id": "db7e968a" + }, + "source": [ + "\"PyPI\n", + "\"PyPI\n", + "\"PyPI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ba87ca5", + "metadata": { + "id": "2ba87ca5" + }, + "outputs": [], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install -q weightslab\n", + "%pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev7\"\n", + "%pip install -q torchmetrics" + ] + }, + { + "cell_type": "markdown", + "id": "b07a3f07", + "metadata": { + "id": "b07a3f07" + }, + "source": [ + "### Fetch the dataset from Google Drive\n", + "\n", + "Set **`DATASET_URL`** below to your Drive **share link** for the dataset **zip** (the file must be shared as *Anyone with the link*). The cell downloads it with [`gdown`](https://github.com/wkentaro/gdown), extracts it, and auto-detects the dataset root — the folder that contains an `images/` subdirectory (so it works even if your zip wraps everything in a top-level folder). `config[\"data_root\"]` then points at that folder.\n", + "\n", + "The zip should unpack to the BDD100k layout the dataset class expects:\n", + "\n", + "```\n", + "/\n", + " images/{train,val}/ # .jpg / .png inputs\n", + " labels/{train,val}/ # matching .png masks (same basename as the image)\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe6b5769", + "metadata": { + "id": "fe6b5769" + }, + "outputs": [], + "source": [ + "# ===== Dataset — local copy preferred (byte-identical to wl-seg-wow-moment-bdd), Drive as fallback =====\n", + "# This notebook is set up to reproduce the \"wl-seg-wow-moment-bdd\" experiment.\n", + "# If that experiment's exact dataset is available locally, use it directly so\n", + "# results are directly comparable. Otherwise (e.g. on Colab) fall back to the\n", + "# Drive zip below (a smaller toy dataset with the same layout/format).\n", + "LOCAL_DATA_ROOT = \"/home/ubuntu/guillaume_plg/data/bdd/bdd8k\" # matches wl-seg-wow-moment-bdd/config.yaml's data_root\n", + "\n", + "# Paste your Drive SHARE link for the dataset zip here (shared as \"Anyone with the link\").\n", + "DATASET_URL = \"https://drive.google.com/file/d/1rXNsROVeneXp91VccOGIjtYWkkhWECBb/view?usp=sharing\"\n", + "\n", + "import os, sys, subprocess, zipfile\n", + "\n", + "\n", + "def _find_data_root(base):\n", + " \"\"\"Return the folder that holds an `images/` subdir (handles a wrapping top-level dir).\"\"\"\n", + " if os.path.isdir(os.path.join(base, \"images\")):\n", + " return base\n", + " for d, subdirs, _ in os.walk(base):\n", + " if \"images\" in subdirs:\n", + " return d\n", + " return base\n", + "\n", + "\n", + "if os.path.isdir(os.path.join(LOCAL_DATA_ROOT, \"images\")):\n", + " print(f\"Found the wl-seg-wow-moment-bdd dataset locally at {LOCAL_DATA_ROOT} — using it directly.\")\n", + " DATA_ROOT = LOCAL_DATA_ROOT\n", + "else:\n", + " # gdown handles Drive downloads (incl. the large-file virus-scan confirmation).\n", + " subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"gdown\"], check=True)\n", + " import gdown\n", + "\n", + " zip_path = \"dataset.zip\"\n", + " extract_dir = \"dataset\"\n", + "\n", + " # Download (fuzzy=True lets gdown accept a full \"file/d//view\" share URL).\n", + " if not os.path.exists(zip_path):\n", + " gdown.download(url=DATASET_URL, output=zip_path, quiet=False, fuzzy=True)\n", + " else:\n", + " print(f\"{zip_path} already present — skipping download.\")\n", + "\n", + " # Extract.\n", + " with zipfile.ZipFile(zip_path) as zf:\n", + " zf.extractall(extract_dir)\n", + "\n", + " DATA_ROOT = _find_data_root(extract_dir)\n", + "\n", + "print(\"Dataset root:\", DATA_ROOT)\n", + "print(\"Contents:\", sorted(os.listdir(DATA_ROOT)))\n", + "for _split in (\"train\", \"val\", \"test\"):\n", + " _p = os.path.join(DATA_ROOT, \"images\", _split)\n", + " if os.path.isdir(_p):\n", + " print(f\" images/{_split}: {len(os.listdir(_p))} files\")" + ] + }, + { + "cell_type": "markdown", + "id": "7feccf69", + "metadata": { + "id": "7feccf69" + }, + "source": [ + "## 1. Imports\n", + "\n", + "`weightslab` is imported as `wl`. The two `guard_*_context` managers scope a block as training vs. evaluation so signals are attributed to the right phase. These are the external imports gathered from the example's `main.py` and `utils/` modules." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39ef03a8", + "metadata": { + "id": "39ef03a8" + }, + "outputs": [], + "source": [ + "import os\n", + "os.environ['WEIGHTSLAB_LOG_LEVEL'] = 'DEBUG'\n", + "import time\n", + "import logging\n", + "import tempfile\n", + "\n", + "import numpy as np\n", + "import tqdm\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from torch import optim\n", + "from torch.utils.data import Dataset\n", + "from torchvision import transforms\n", + "from PIL import Image\n", + "\n", + "import weightslab as wl\n", + "from weightslab.components.global_monitoring import (\n", + " guard_training_context,\n", + " guard_testing_context,\n", + ")\n", + "\n", + "# Setup loggers (from main.py)\n", + "logging.basicConfig(level=logging.ERROR)\n", + "logging.getLogger(\"PIL\").setLevel(logging.INFO)\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "markdown", + "id": "cef2413d", + "metadata": { + "id": "cef2413d" + }, + "source": [ + "## 2. The dataset (inlined `utils/data.py`)\n", + "\n", + "`BDD100kSegDataset` returns `(image, uid, mask, metadata)` per sample, where `mask` is a single `[H, W]` semantic mask (pixel value = class id). `seg_collate` stacks the batch into `images [B, C, H, W]` and `labels [B, H, W]` so signals are attributed per sample." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2413697c", + "metadata": { + "id": "2413697c" + }, + "outputs": [], + "source": [ + "# ===== utils/data.py — SAMPLE-WISE semantic segmentation =====\n", + "import os\n", + "import torch\n", + "import numpy as np\n", + "\n", + "from torchvision import transforms\n", + "\n", + "from torch.utils.data import Dataset\n", + "\n", + "from PIL import Image\n", + "\n", + "\n", + "# =============================================================================\n", + "# BDD100k segmentation dataset (sample-wise)\n", + "# =============================================================================\n", + "class BDD100kSegDataset(Dataset):\n", + " \"\"\"Returns (image, uid, mask, metadata) with ONE semantic mask per sample.\n", + "\n", + " Layout expected under `root`:\n", + " images/{train,val}/ inputs (.jpg/.png)\n", + " labels/{train,val}/ masks (.png, same basename; pixel value = class id)\n", + "\n", + " `mask` is a single [H, W] int64 tensor of class ids (0..num_classes-1;\n", + " `ignore_index` for void). No per-instance masks — signals are per sample.\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " root,\n", + " split=\"train\",\n", + " num_classes=6,\n", + " class_names=None,\n", + " ignore_index=255,\n", + " image_size=256,\n", + " max_samples=None\n", + " ):\n", + " super().__init__()\n", + " self.root = root\n", + " self.split = split\n", + " self.num_classes = num_classes\n", + " self.class_names = class_names\n", + " self.ignore_index = ignore_index\n", + " self.task_type = \"segmentation\"\n", + "\n", + " img_dir = os.path.join(root, \"images\", split)\n", + " lbl_dir = os.path.join(root, \"labels\", split)\n", + "\n", + " image_files = [\n", + " f\n", + " for f in os.listdir(img_dir)\n", + " if f.lower().endswith((\".jpg\", \".jpeg\", \".png\"))\n", + " ]\n", + " image_files = sorted(set(image_files))[:max_samples] if max_samples != None else sorted(set(image_files)) # Optionally limit number of samples for faster testing\n", + "\n", + " self.images = []\n", + " self.masks = []\n", + " for fname in image_files:\n", + " img_path = os.path.join(img_dir, fname)\n", + " base, _ = os.path.splitext(fname)\n", + " lbl_name = base + \".png\"\n", + " lbl_path = os.path.join(lbl_dir, lbl_name)\n", + " if os.path.exists(lbl_path):\n", + " self.images.append(img_path)\n", + " self.masks.append(lbl_path)\n", + "\n", + " if len(self.images) == 0:\n", + " raise RuntimeError(f\"No image/label pairs found in {img_dir} / {lbl_dir}\")\n", + "\n", + " # Resize to a fixed SQUARE (image_size, image_size). A single int would\n", + " # resize only the shorter edge and keep the aspect ratio, so non-square\n", + " # inputs come out as e.g. [128, 227] and mismatch the prediction in the\n", + " # loss. The model input is square (input_shape=(1, C, image_size, image_size)).\n", + " self.image_transform = transforms.Compose(\n", + " [\n", + " transforms.Resize(\n", + " # size=(image_size, image_size),\n", + " size=image_size, # single int = resize shorter edge, keep aspect ratio\n", + " interpolation=Image.BILINEAR\n", + " ),\n", + " transforms.ToTensor(),\n", + " ]\n", + " )\n", + " self.mask_resize = transforms.Resize(\n", + " # size=(image_size, image_size),\n", + " size=image_size,\n", + " interpolation=Image.NEAREST\n", + " )\n", + "\n", + " def __len__(self):\n", + " return len(self.images)\n", + "\n", + " def __getitem__(self, idx):\n", + " \"\"\"Returns (item, uid, target, metadata):\n", + " - item: input image tensor [C, H, W]\n", + " - uid: unique id for the sample (filename)\n", + " - target: semantic mask tensor [H, W] (int64 class ids)\n", + " - metadata: optional dict (original paths)\n", + " \"\"\"\n", + " return self.get_items(idx, include_metadata=True, include_labels=True, include_images=True)\n", + "\n", + " def get_items(self, idx, include_metadata=False, include_labels=False, include_images=False):\n", + " img_path = self.images[idx]\n", + " mask_path = self.masks[idx]\n", + " uid = os.path.basename(img_path)\n", + "\n", + " metadata = {\n", + " 'img_path': img_path,\n", + " 'mask_path': mask_path,\n", + " } if include_metadata else None\n", + "\n", + " # Process image\n", + " img_t = None\n", + " if include_images:\n", + " img = Image.open(img_path).convert(\"RGB\")\n", + " img_t = self.image_transform(img)\n", + "\n", + " # Process label: one semantic mask [H, W] per sample.\n", + " mask_t = None\n", + " if include_labels:\n", + " mask = Image.open(mask_path)\n", + " mask_r = self.mask_resize(mask)\n", + " mask_np = np.array(mask_r, dtype=np.int64)\n", + " # Mask must be a single-channel class-id map [H, W]. If the file is\n", + " # RGB/RGBA (extra channel dim), keep the first channel so it matches\n", + " # the model's [H, W] prediction.\n", + " if mask_np.ndim > 2:\n", + " mask_np = mask_np[..., 0]\n", + " mask_t = torch.from_numpy(mask_np) # [H, W] int64\n", + "\n", + " return img_t, uid, mask_t, metadata\n", + "\n", + "\n", + "def seg_collate(batch):\n", + " \"\"\"Collate WL per-sample tuples for semantic segmentation.\n", + "\n", + " Each item is ``(img, uid, mask, metadata)`` where ``mask`` is a single\n", + " [H, W] class-id tensor. Images and masks stack into batched tensors; uids\n", + " and metadata stay as per-sample lists.\n", + "\n", + " Returns:\n", + " images: FloatTensor [B, C, H, W]\n", + " ids: list[str] of length B\n", + " labels: LongTensor [B, H, W]\n", + " metas: list[B] of metadata dicts\n", + " \"\"\"\n", + " images = torch.stack([b[0] for b in batch], dim=0)\n", + " ids = [b[1] for b in batch]\n", + " labels = torch.stack([torch.as_tensor(b[2]) for b in batch], dim=0)\n", + " metas = [b[3] if len(b) > 3 else None for b in batch]\n", + " return images, ids, labels, metas" + ] + }, + { + "cell_type": "markdown", + "id": "352304ca", + "metadata": { + "id": "352304ca" + }, + "source": [ + "## 3. The model (inlined `utils/model.py`)\n", + "\n", + "A compact U-Net (`SmallUNet`) with `task_type=\"segmentation\"` so WeightsLab renders masks in Weights Studio." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ecaac87", + "metadata": { + "id": "0ecaac87" + }, + "outputs": [], + "source": [ + "# ===== utils/model.py =====\n", + "# =============================================================================\n", + "# Small UNet-ish segmentation model\n", + "# =============================================================================\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "\n", + "\n", + "class DoubleConv(nn.Module):\n", + " def __init__(self, in_ch, out_ch):\n", + " super().__init__()\n", + " self.block = nn.Sequential(\n", + " nn.Conv2d(in_ch, out_ch, 3, padding=1),\n", + " nn.BatchNorm2d(out_ch),\n", + " nn.ReLU(inplace=True),\n", + " nn.Conv2d(out_ch, out_ch, 3, padding=1),\n", + " nn.BatchNorm2d(out_ch),\n", + " nn.ReLU(inplace=True),\n", + " )\n", + "\n", + " def forward(self, x):\n", + " return self.block(x)\n", + "\n", + "\n", + "class SmallUNet(nn.Module):\n", + " def __init__(self, in_channels=3, num_classes=6, image_size=256):\n", + " super().__init__()\n", + " # For WeightsLab\n", + " self.task_type = \"segmentation\"\n", + " self.num_classes = num_classes\n", + " self.class_names = [\"Background\", \"Ego Road\", \"Driveable Area\", \"Lane Line 1\", \"Lane Line 2\", \"Lane Line 3\"]\n", + " self.input_shape = (1, in_channels, image_size, image_size)\n", + "\n", + " self.enc1 = DoubleConv(in_channels, 32)\n", + " self.pool1 = nn.MaxPool2d(2)\n", + " self.enc2 = DoubleConv(32, 64)\n", + " self.pool2 = nn.MaxPool2d(2)\n", + "\n", + " self.bottleneck = DoubleConv(64, 128)\n", + "\n", + " self.up2 = nn.ConvTranspose2d(128, 64, 2, stride=2)\n", + " self.dec2 = DoubleConv(64 + 64, 64)\n", + " self.up1 = nn.ConvTranspose2d(64, 32, 2, stride=2)\n", + " self.dec1 = DoubleConv(32 + 32, 32)\n", + "\n", + " self.head = nn.Conv2d(32, num_classes, 1)\n", + "\n", + " def forward(self, x):\n", + " # Encoder\n", + " e1 = self.enc1(x)\n", + " p1 = self.pool1(e1)\n", + "\n", + " e2 = self.enc2(p1)\n", + " p2 = self.pool2(e2)\n", + "\n", + " # Bottleneck\n", + " b = self.bottleneck(p2)\n", + "\n", + " # Decoder\n", + " u2 = self.up2(b)\n", + " # Important: no `if` on shapes; always interpolate\n", + " u2 = F.interpolate(u2, size=e2.shape[-2:], mode=\"bilinear\", align_corners=False)\n", + " d2 = self.dec2(torch.cat([u2, e2], dim=1))\n", + "\n", + " u1 = self.up1(d2)\n", + " u1 = F.interpolate(u1, size=e1.shape[-2:], mode=\"bilinear\", align_corners=False)\n", + " d1 = self.dec1(torch.cat([u1, e1], dim=1))\n", + "\n", + " logits = self.head(d1) # [B, C, H, W]\n", + " return logits" + ] + }, + { + "cell_type": "markdown", + "id": "7d382efc", + "metadata": { + "id": "7d382efc" + }, + "source": [ + "## 4. Losses & metrics (inlined `utils/criterions.py`)\n", + "\n", + "Per-sample combined **CrossEntropy + Dice** (loss) and **Dice** (metric), ported to match the `wl-seg-wow-moment-bdd` experiment's `criterions.py::CrossEntropySegLoss` exactly (same formula, same per-class weights, same `ignore_index` handling — see that class's docstring below for a quirk in the reference project that's preserved here for faithful reproduction). Each scores the model's `[B, C, H, W]` output against the `[B, H, W]` mask and returns **one value per sample** (`[B]`), wrapped with `per_sample=True` so WeightsLab logs it per `sample_id`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c483178e", + "metadata": { + "id": "c483178e" + }, + "outputs": [], + "source": [ + "# ===== utils/criterions.py — SAMPLE-WISE Dice (metric) + combined CE+Dice (loss) =====\n", + "# The loss below is ported to match wl-seg-wow-moment-bdd/criterions.py's\n", + "# CrossEntropySegLoss exactly (same formula, same per-class weighting, same\n", + "# ignore_index handling) so this notebook reproduces that experiment's loss.\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "\n", + "_EPS = 1e-7 # matches PerSampleMultiClassDiceLoss's eps in wl-seg-wow-moment-bdd/criterions.py\n", + "\n", + "\n", + "class PerSampleMultiClassDiceLoss(nn.Module):\n", + " \"\"\"Per-sample, per-class Dice LOSS (1 - diceScore), averaged over classes -> [B].\n", + "\n", + " Ported verbatim (same formula/eps/masking) from\n", + " wl-seg-wow-moment-bdd/criterions.py::PerSampleMultiClassDiceLoss.\n", + " \"\"\"\n", + "\n", + " def __init__(self, ignore_index=255, eps=_EPS, weights=None, ignore_classes=None):\n", + " super().__init__()\n", + " self.ignore_index = ignore_index\n", + " self.eps = eps\n", + " self.weights = weights\n", + " self.ignore_classes = ignore_classes\n", + "\n", + " def forward(self, logits, targets):\n", + " bs, num_classes, H, W = logits.shape\n", + " probs = torch.softmax(logits, dim=1) # [B, C, H, W]\n", + " valid = (targets != self.ignore_index) # [B, H, W]\n", + " targets_safe = targets.clone()\n", + " targets_safe[~valid] = 0\n", + " one_hot = F.one_hot(targets_safe, num_classes=num_classes).permute(0, 3, 1, 2).float()\n", + "\n", + " valid_f = valid.unsqueeze(1).float() # [B, 1, H, W]\n", + " probs = probs * valid_f\n", + " one_hot = one_hot * valid_f\n", + "\n", + " intersection = (probs * one_hot).sum(dim=(2, 3)) # [B, C]\n", + " cardinality = (probs + one_hot).sum(dim=(2, 3)) # [B, C]\n", + " dice_per_class = 1.0 - (2.0 * intersection + self.eps) / (cardinality + self.eps) # [B, C] — a LOSS, not a score\n", + "\n", + " if self.ignore_classes:\n", + " class_mask = torch.ones(num_classes, dtype=torch.bool, device=logits.device)\n", + " class_mask[list(self.ignore_classes)] = False\n", + " dice_per_class = dice_per_class[:, class_mask]\n", + " weights = self.weights[class_mask] if self.weights is not None else None\n", + " else:\n", + " weights = self.weights\n", + "\n", + " if weights is not None:\n", + " dice_per_class = dice_per_class * weights\n", + " return dice_per_class.mean(dim=1) # [B]\n", + "\n", + "\n", + "class PerSampleCEDice(nn.Module):\n", + " \"\"\"Combined CrossEntropy + Dice loss -> [B] (differentiable, per-sample).\n", + "\n", + " Ported from wl-seg-wow-moment-bdd/criterions.py::CrossEntropySegLoss, with\n", + " one fix applied on both sides: `PerSampleMultiClassDiceLoss` above already\n", + " returns a dice LOSS (`1 - diceScore`), so it's used directly here. The\n", + " reference project originally did `(1 - dice)` again on top of that, which\n", + " algebraically reduces to the raw dice SCORE (i.e. a *better* dice score\n", + " increased the loss) — that double-negation has been fixed in both this\n", + " notebook and wl-seg-wow-moment-bdd/criterions.py.\n", + " \"\"\"\n", + "\n", + " def __init__(self, weights=None, ignore_index=255, bce_weight=0.5, dice_weight=0.5):\n", + " super().__init__()\n", + " self.register_buffer(\n", + " \"weights\",\n", + " torch.as_tensor(weights, dtype=torch.float32) if weights is not None else None,\n", + " )\n", + " self.ignore_index = ignore_index\n", + " self.bce_weight = bce_weight\n", + " self.dice_weight = dice_weight\n", + " self.dice = PerSampleMultiClassDiceLoss(ignore_index=ignore_index, weights=self.weights)\n", + "\n", + " def forward(self, outputs, labels):\n", + " # outputs [B, C, H, W]; labels [B, H, W] (int64 class ids)\n", + " ce = F.cross_entropy(\n", + " outputs, labels.long(),\n", + " weight=self.weights, ignore_index=self.ignore_index, reduction=\"none\",\n", + " ) # [B, H, W]\n", + " # Mean over ALL pixels (ignored ones included as 0), matching the\n", + " # source project's un-masked mean (not a valid-pixel-only average).\n", + " ce = ce.flatten(1).mean(dim=1) # [B]\n", + "\n", + " dice_loss = self.dice(outputs, labels.long()) # [B] — already a LOSS (1 - diceScore)\n", + "\n", + " return self.bce_weight * ce + self.dice_weight * dice_loss\n", + "\n", + "\n", + "class PerSampleDice(nn.Module):\n", + " \"\"\"Per-sample mean foreground Dice -> [B] (metric, detached; for the WL dashboard).\"\"\"\n", + "\n", + " def forward(self, outputs, labels):\n", + " probs = torch.softmax(outputs, dim=1) # [B, C, H, W]\n", + " C = probs.shape[1]\n", + " tgt = torch.where(labels < C, labels, torch.zeros_like(labels)).long() # void/ignore -> background\n", + " onehot = F.one_hot(tgt, num_classes=C).permute(0, 3, 1, 2).float() # [B, C, H, W]\n", + " inter = (probs * onehot).sum(dim=(2, 3))\n", + " denom = probs.sum(dim=(2, 3)) + onehot.sum(dim=(2, 3))\n", + " dice = (2.0 * inter + _EPS) / (denom + _EPS) # [B, C]\n", + " fg = dice[:, 1:].mean(dim=1) if C > 1 else dice.mean(dim=1) # exclude background channel\n", + " return fg.detach() # [B]" + ] + }, + { + "cell_type": "markdown", + "id": "2965ed80", + "metadata": { + "id": "2965ed80" + }, + "source": [ + "## 5. Configuration\n", + "\n", + "Every tunable lives here in one dict - this is `config.yaml` inlined, with its comments. Wrapping it with `flag=\"hyperparameters\"` lets Weights Studio read (and live-edit) these values while training. `data_root` points at the sample fetched above.\n", + "\n", + "**Reproducing `wl-seg-wow-moment-bdd`:** `ignore_index`, `image_size`, `training_steps_to_do`, `eval_full_to_train_steps_ratio`, and the train `batch_size` below are set to match that experiment's `config.yaml` exactly, since these directly change training results (model architecture, optimizer, and dataset format were already identical)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00e4c364", + "metadata": { + "id": "00e4c364" + }, + "outputs": [], + "source": [ + "config = {\n", + " # -- Global configuration ------------------------------------------------\n", + " \"device\": \"auto\", # 'auto' -> resolved to `device` from the Imports cell\n", + " \"experiment_name\": \"seg_bdd_training\", # name shown in Weights Studio\n", + " \"training_steps_to_do\": 10000, # matches wl-seg-wow-moment-bdd/config.yaml (training_steps_to_do)\n", + " # \"root_log_dir\": \"/tmp/tmpg947ov7y/\", # None -> a temp dir is created (history + dataframe land here)\n", + "\n", + " \"checkpoint_manager\": {\n", + " \"load_config\": False, # don't reload a previous checkpoint's config, so edits here take effect\n", + " },\n", + "\n", + " # Initially compute natural sorting values, e.g. average intensity (see the example README).\n", + " \"compute_natural_sort\": True,\n", + "\n", + " # -- Experiment parameters ----------------------------------------------\n", + " \"eval_full_to_train_steps_ratio\": 256, # matches wl-seg-wow-moment-bdd (test every 256 steps)\n", + " \"experiment_dump_to_train_steps_ratio\": 256,\n", + " \"write_export_ratio\": 100, # export history + dataframe every N steps (main.py default)\n", + " \"tqdm_display\": True,\n", + " \"is_training\": False, # start training immediately or not\n", + "\n", + " # -- Serving (the notebook serves via gRPC + a bore tunnel; see the Serve cell) --\n", + " \"serving_grpc\": True,\n", + " \"serving_cli\": True,\n", + "\n", + " # -- Global dataframe storage -------------------------------------------\n", + " \"ledger_enable_h5_persistence\": True,\n", + " \"ledger_enable_flushing_threads\": True,\n", + " \"ledger_flush_max_rows\": 100,\n", + " \"ledger_flush_interval\": 60.0,\n", + "\n", + " # -- Data ----------------------------------------------------------------\n", + " \"num_classes\": 6,\n", + " \"class_names\": [\"Background\", \"Ego Road\", \"Driveable Area\", \"Lane Line 1\", \"Lane Line 2\", \"Lane Line 3\"],\n", + " \"ignore_index\": 0, # matches wl-seg-wow-moment-bdd: excludes \"Background\" (class 0) from loss/metric\n", + " \"image_size\": 180, # matches wl-seg-wow-moment-bdd (shortest edge resized to 720, aspect ratio kept)\n", + " \"data_root\": DATA_ROOT, # local wl-seg-wow-moment-bdd dataset if found, else the Drive zip (see the data-fetch cell)\n", + " \"data\": {\n", + " \"train_loader\": {\"batch_size\": 8, \"shuffle\": True}, # matches wl-seg-wow-moment-bdd (train batch_size 8)\n", + " \"test_loader\": {\"batch_size\": 2, \"shuffle\": False},\n", + " },\n", + "\n", + " # -- Optimizer -----------------------------------------------------------\n", + " \"optimizer\": {\"lr\": 1e-3}, # Adam learning rate (matches wl-seg-wow-moment-bdd, which has no `optimizer:` key so falls back to this same 1e-3 default)\n", + "}\n", + "\n", + "# Resolve the 'auto' device to the one picked in the Imports cell.\n", + "if config.get(\"device\", \"auto\") == \"auto\":\n", + " config[\"device\"] = str(device)\n", + "\n", + "# Register the config so Weights Studio can read (and live-edit) it while training.\n", + "wl.watch_or_edit(config, flag=\"hyperparameters\", name=config[\"experiment_name\"],\n", + " defaults=config, poll_interval=1.0)\n", + "\n", + "# Logging directory: None -> a temp dir is created.\n", + "if not config.get(\"root_log_dir\"):\n", + " config[\"root_log_dir\"] = tempfile.mkdtemp(prefix=\"weightslab_seg_\")\n", + "os.makedirs(config[\"root_log_dir\"], exist_ok=True)\n", + "log_dir = config[\"root_log_dir\"]\n", + "\n", + "# Schedule knobs used by the training loop.\n", + "eval_full_to_train_steps_ratio = config[\"eval_full_to_train_steps_ratio\"]\n", + "write_export_ratio = config.get(\"write_export_ratio\", 100)\n", + "tqdm_display = config.get(\"tqdm_display\", True)\n", + "print(\"Experiment logs ->\", log_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "88cc2afa", + "metadata": { + "id": "88cc2afa" + }, + "source": [ + "## 6. Build data, model & optimizer\n", + "\n", + "The heart of WeightsLab: each object is passed through `wl.watch_or_edit(...)` with a `flag` describing its role. The returned objects behave like the originals but report their state and per-sample signals. The loaders use `collate_fn=seg_collate` to stack the image and mask batches." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9df9c8cf", + "metadata": { + "id": "9df9c8cf" + }, + "outputs": [], + "source": [ + "# --- Hyperparameters read back after watch_or_edit ---\n", + "num_classes = int(config[\"num_classes\"])\n", + "class_names = config[\"class_names\"]\n", + "ignore_index = int(config[\"ignore_index\"])\n", + "image_size = int(config[\"image_size\"])\n", + "\n", + "# --- Data (local wl-seg-wow-moment-bdd dataset, or the Drive zip fetched above) ---\n", + "data_root = config.get(\"data_root\", DATA_ROOT)\n", + "if os.path.exists(data_root):\n", + " print(f\"Using data root: {data_root}\")\n", + "else:\n", + " raise FileNotFoundError(\n", + " f\"Data root not found: {data_root!r}. Run the data-fetch cell above first.\")\n", + "\n", + "train_cfg = config.get(\"data\", {}).get(\"train_loader\", {})\n", + "test_cfg = config.get(\"data\", {}).get(\"test_loader\", {})\n", + "\n", + "# wl-seg-wow-moment-bdd's dataset uses images/{train,test}; the packaged Drive\n", + "# toy dataset uses images/{train,val} — detect whichever is actually present.\n", + "eval_split = \"test\" if os.path.isdir(os.path.join(data_root, \"images\", \"test\")) else \"val\"\n", + "print(f\"Eval split: {eval_split!r}\")\n", + "\n", + "_train_dataset = BDD100kSegDataset(\n", + " root=data_root,\n", + " split=\"train\",\n", + " num_classes=num_classes,\n", + " class_names=class_names,\n", + " ignore_index=ignore_index,\n", + " image_size=image_size,\n", + " max_samples=train_cfg.get(\"max_samples\", None),\n", + ")\n", + "_test_dataset = BDD100kSegDataset(\n", + " root=data_root,\n", + " split=eval_split,\n", + " num_classes=num_classes,\n", + " class_names=class_names,\n", + " ignore_index=ignore_index,\n", + " image_size=image_size,\n", + " max_samples=test_cfg.get(\"max_samples\", None),\n", + ")\n", + "\n", + "train_loader = wl.watch_or_edit(\n", + " _train_dataset, flag=\"data\", loader_name=\"train_loader\",\n", + " batch_size=train_cfg.get(\"batch_size\", 2), shuffle=train_cfg.get(\"shuffle\", True),\n", + " compute_hash=False, is_training=True,\n", + " array_autoload_arrays=False, array_return_proxies=True, array_use_cache=True,\n", + " preload_labels=False, collate_fn=seg_collate,\n", + ")\n", + "test_loader = wl.watch_or_edit(\n", + " _test_dataset, flag=\"data\", loader_name=\"test_loader\",\n", + " batch_size=test_cfg.get(\"batch_size\", 2), shuffle=test_cfg.get(\"shuffle\", False),\n", + " compute_hash=False, is_training=False,\n", + " array_autoload_arrays=False, array_return_proxies=True, array_use_cache=True,\n", + " preload_labels=True, collate_fn=seg_collate,\n", + ")\n", + "\n", + "_model = SmallUNet(in_channels=3, num_classes=num_classes, image_size=image_size).to(device)\n", + "model = wl.watch_or_edit(_model, flag=\"model\", device=device, compute_dependencies=True)\n", + "\n", + "lr = config.get(\"optimizer\", {}).get(\"lr\", 1e-3)\n", + "_optimizer = optim.Adam(model.parameters(), lr=lr)\n", + "optimizer = wl.watch_or_edit(_optimizer, flag=\"optimizer\")" + ] + }, + { + "cell_type": "markdown", + "id": "ec7508f0", + "metadata": { + "id": "ec7508f0" + }, + "source": [ + "## 7. Train & eval steps (+ tracked signals)\n", + "\n", + "The `guard_training_context` / `guard_testing_context` blocks tell WeightsLab which phase it's in. `_run_signals` computes and logs the per-sample **Dice** (metric) and **CrossEntropy** (loss); `wl.save_signals(...)` records custom per-sample diagnostics (predicted vs present classes). Class weights are computed from the training split, then the tracked signals are built." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52c07147", + "metadata": { + "id": "52c07147" + }, + "outputs": [], + "source": [ + "# ===== train / eval step functions (sample-wise) =====\n", + "from torchmetrics import JaccardIndex\n", + "\n", + "\n", + "def _run_signals(sig, outputs, labels, ids, preds, return_metric=False):\n", + " \"\"\"Compute + log the per-sample combined CE+Dice (loss) and Dice (metric).\"\"\"\n", + " loss_sample = sig[\"loss_sample\"](outputs, labels, batch_ids=ids, preds=preds)\n", + " dice_sample = sig[\"dice_sample\"](outputs, labels, batch_ids=ids)\n", + " if return_metric:\n", + " return loss_sample, dice_sample\n", + " return loss_sample\n", + "\n", + "\n", + "def _user_custom_signals(preds, labels):\n", + " \"\"\"Per-sample diagnostics: which classes are predicted vs present in the mask.\"\"\"\n", + " B = preds.shape[0]\n", + " return {\n", + " \"preds_classes_per_sample\": [preds[i].unique() for i in range(B)],\n", + " \"target_classes_per_sample\": [labels[i].unique() for i in range(B)],\n", + " \"tp_classes_per_sample\": [\n", + " torch.tensor([c for c in labels[i].unique() if c in preds[i].unique()]) for i in range(B)\n", + " ],\n", + " \"fp_classes_per_sample\": [\n", + " torch.tensor([c for c in preds[i].unique() if c not in labels[i].unique()]) for i in range(B)\n", + " ],\n", + " \"fn_classes_per_sample\": [\n", + " torch.tensor([c for c in labels[i].unique() if c not in preds[i].unique()]) for i in range(B)\n", + " ],\n", + " }\n", + "\n", + "\n", + "def train(loader, model, optimizer, sig, metric, device):\n", + " \"\"\"Single training step. loader yields (inputs, ids, labels, metadata);\n", + " `labels` is a [B, H, W] semantic mask (see seg_collate).\"\"\"\n", + " with guard_training_context:\n", + " (inputs, ids, labels, _) = next(loader)\n", + " inputs = inputs.to(device)\n", + " labels = labels.to(device) # [B, H, W]\n", + "\n", + " optimizer.zero_grad()\n", + " outputs = model(inputs) # [B, C, H, W]\n", + " preds = outputs.argmax(dim=1) # [B, H, W]\n", + "\n", + " # Per-sample combined CE+Dice (loss) + Dice (metric), tracked & saved per sample.\n", + " loss_per_sample = _run_signals(sig, outputs, labels, ids, preds=preds) # [B]\n", + " loss = loss_per_sample.mean()\n", + "\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # Per-sample class diagnostics for the UI (predicted vs present classes).\n", + " wl.save_signals(_user_custom_signals(preds, labels), ids)\n", + "\n", + " # Batch IoU, matching wl-seg-wow-moment-bdd's train() (reset every step).\n", + " metric.update(preds, labels.detach())\n", + " train_iou = metric.compute().detach().cpu().item() * 100.0\n", + " metric.reset()\n", + "\n", + " return float(loss.detach().cpu().item()), train_iou\n", + "\n", + "\n", + "def test(loader, model, sig, metric, device, test_loader_len):\n", + " \"\"\"Full evaluation pass over the eval loader.\"\"\"\n", + " losses = torch.tensor([0.]).to(device)\n", + " dices = torch.tensor([0.]).to(device)\n", + " metric.reset()\n", + " with guard_testing_context, torch.no_grad():\n", + " for inputs, ids, labels, _ in loader:\n", + " inputs = inputs.to(device)\n", + " labels = labels.to(device) # [B, H, W]\n", + "\n", + " outputs = model(inputs)\n", + " preds = outputs.argmax(dim=1) # [B, H, W]\n", + "\n", + " loss_per_sample, dice_sample = _run_signals(sig, outputs, labels, ids, preds=preds, return_metric=True)\n", + " losses += torch.mean(loss_per_sample) # accumulate batch mean\n", + " dices += torch.mean(dice_sample)\n", + " metric.update(preds, labels)\n", + "\n", + " wl.save_signals(_user_custom_signals(preds, labels), ids)\n", + "\n", + " loss = float((losses / test_loader_len).detach().cpu().item())\n", + " dice = float((dices / test_loader_len).detach().cpu().item())\n", + " iou = metric.compute().detach().cpu().item() * 100.0\n", + " return loss, dice * 100.0, iou # Dice/IoU as percentages\n", + "\n", + "\n", + "# ===== tracked per-sample Dice (metric) + combined CE+Dice (loss) signals =====\n", + "# Both are wrapped with per_sample=True so WeightsLab logs one value per\n", + "# sample_id. No per-instance signals.\n", + "def _make_seg_signals(split: str, weights=None, ignore_index: int = 255) -> dict:\n", + " return {\n", + " \"dice_sample\": wl.watch_or_edit(\n", + " PerSampleDice(), flag=\"metric\",\n", + " name=f\"{split}_dice/sample\", per_sample=True, log=True,\n", + " ),\n", + " \"loss_sample\": wl.watch_or_edit(\n", + " PerSampleCEDice(weights=weights, ignore_index=ignore_index), flag=\"loss\",\n", + " name=f\"{split}_CE\", per_sample=True, log=True,\n", + " ),\n", + " }\n", + "\n", + "\n", + "def compute_class_weights(dataset, num_classes, ignore_index=255, max_samples=100):\n", + " print(\"\\n\" + \"=\" * 60, flush=True)\n", + " print(f\"Computing class weights for {num_classes} classes (max {max_samples} samples)...\", flush=True)\n", + " class_counts = np.zeros(num_classes, dtype=np.float64)\n", + " num_samples = min(len(dataset), max_samples)\n", + "\n", + " for idx in tqdm.tqdm(range(num_samples), desc=\" Analyzing Distribution\"):\n", + " _, _, label, _ = dataset.get_items(idx, include_labels=True) # semantic mask [H, W]\n", + " label_np = label.numpy() if hasattr(label, 'numpy') else np.array(label)\n", + " for c in range(num_classes):\n", + " class_counts[c] += (label_np == c).sum()\n", + "\n", + " class_counts = np.maximum(class_counts, 1) # Avoid div by zero\n", + " total_pixels = class_counts.sum()\n", + " class_weights = total_pixels / (num_classes * class_counts)\n", + " class_weights = class_weights / class_weights.mean() # Normalize\n", + "\n", + " print(\"\\nClass distribution and weights:\", flush=True)\n", + " for c in range(num_classes):\n", + " pct = (class_counts[c] / total_pixels) * 100\n", + " print(f\"Class {c}: {pct:6.2f}% -> weight: {class_weights[c]:.3f}\", flush=True)\n", + " print(\"=\" * 60 + \"\\n\", flush=True)\n", + " return torch.FloatTensor(class_weights).to(device)\n", + "\n", + "weights = compute_class_weights(_train_dataset, num_classes, max_samples=100)\n", + "\n", + "# Matches wl-seg-wow-moment-bdd: the train criterion is class-weighted, the\n", + "# test criterion is NOT (its CrossEntropySegLoss gets no `weights=` argument there).\n", + "train_sig = _make_seg_signals(\"train\", weights=weights, ignore_index=ignore_index)\n", + "test_sig = _make_seg_signals(\"test\", weights=None, ignore_index=ignore_index)\n", + "\n", + "# IoU (macro, ignore_index=0), matching wl-seg-wow-moment-bdd's train_iou/test_iou.\n", + "train_metric = wl.watch_or_edit(\n", + " JaccardIndex(task=\"multiclass\", num_classes=num_classes, ignore_index=ignore_index, average=\"macro\").to(device),\n", + " flag=\"metric\", name=\"train_iou\", per_sample=False, log=True,\n", + ")\n", + "test_metric = wl.watch_or_edit(\n", + " JaccardIndex(task=\"multiclass\", num_classes=num_classes, ignore_index=ignore_index, average=\"macro\").to(device),\n", + " flag=\"metric\", name=\"test_iou\", per_sample=False, log=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "fb9890dd", + "metadata": { + "id": "fb9890dd" + }, + "source": [ + "## 8. Serve and train\n", + "\n", + "`wl.serve(serving_grpc=True, serving_bore=True)` starts the background gRPC server and opens a public `bore.pub:` tunnel (printed below) that Weights Studio connects to. `wl.start_training(...)` flips the experiment into the *training* state, then we run a finite loop, periodically evaluating and exporting signals.\n", + "\n", + "Leave this cell **running** while you watch it stream in the UI." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "635014b7", + "metadata": { + "collapsed": true, + "id": "635014b7" + }, + "outputs": [], + "source": [ + "wl.serve(serving_grpc=config.get(\"serving_grpc\", True), serving_bore=True)\n", + "# wl.start_training(timeout=60)\n", + "\n", + "# training_steps_to_do may be None (open-ended); cap it here so the notebook run is finite.\n", + "max_steps = config[\"training_steps_to_do\"] or 500\n", + "train_range = tqdm.tqdm(range(max_steps), desc=\"Training\") if tqdm_display else range(max_steps)\n", + "test_loss, test_dice, test_iou = None, None, None\n", + "start_time = time.time()\n", + "for train_step in train_range:\n", + " age = model.get_age() if hasattr(model, \"get_age\") else train_step\n", + "\n", + " # Train one step\n", + " train_loss, train_iou = train(train_loader, model, optimizer, train_sig, train_metric, device)\n", + "\n", + " # Full evaluation pass\n", + " if age == 0 or age % eval_full_to_train_steps_ratio == 0:\n", + " test_loader_len = len(test_loader)\n", + " test_loader_it = tqdm.tqdm(test_loader, desc=\"Evaluating\", leave=False) if tqdm_display else test_loader\n", + " test_loss, test_dice, test_iou = test(test_loader_it, model, test_sig, test_metric, device, test_loader_len)\n", + "\n", + " if tqdm_display:\n", + " train_range.set_description(\"Step\")\n", + " train_range.set_postfix(\n", + " train_loss=f\"{train_loss:.4f}\",\n", + " train_iou=f\"{train_iou:.2f}%\",\n", + " test_loss=f\"{test_loss:.4f}\" if test_loss is not None else \"N/A\",\n", + " test_dice=f\"{test_dice:.2f}%\" if test_dice is not None else \"N/A\",\n", + " test_iou=f\"{test_iou:.2f}%\" if test_iou is not None else \"N/A\",\n", + " )\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(f\" Training completed in {time.time() - start_time:.2f} seconds\")\n", + "print(f\" Logs saved to: {log_dir}\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Final export of signal history and data grid to root_log_dir\n", + "wl.write_history()\n", + "wl.write_dataframe()" + ] + }, + { + "cell_type": "markdown", + "id": "3812a421", + "metadata": { + "id": "3812a421" + }, + "source": [ + "## See it live in Weights Studio\n", + "\n", + "**Colab has no Docker daemon**, so run Studio on your own machine and point it at this notebook's\n", + "backend using the `bore.pub:` printed in the Expose section:\n", + "\n", + "```bash\n", + "pip install weightslab\n", + "weightslab start # Terminal 1 - opens http://localhost:5173\n", + "weightslab tunnel bore.pub:12345 # Terminal 2 - the host:port from the Expose section\n", + "```\n", + "\n", + "Then open **http://localhost:5173**. Prefer local-only? Run this example directly on your machine\n", + "(`weightslab start example` selects a bundled example) and launch the UI next to it - no tunnel." + ] + }, + { + "cell_type": "markdown", + "id": "36c018d3", + "metadata": { + "id": "36c018d3" + }, + "source": [ + "---\n", + "\n", + "
\n", + "Crafted by GrayboxTech - if WeightsLab helps you catch a bad label, drop us a star on GitHub.\n", + "
" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "wl-segmentation.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-brain-tumor-detection-dataset.ipynb b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-brain-tumor-detection-dataset.ipynb new file mode 100644 index 00000000..dd719903 --- /dev/null +++ b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-brain-tumor-detection-dataset.ipynb @@ -0,0 +1,1908 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Jh24MV9yKs7J" + }, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Train an Ultralytics YOLO detector on the MRI brain-tumor dataset and stream every sample, prediction and metric live into Weights Studio to inspect, edit and evolve your run.
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ImSwebklKs7S" + }, + "source": [ + "# Brain Tumor Detection (MRI) · Ultralytics YOLO + WeightsLab\n", + "\n", + "This notebook trains a YOLO detector on the [brain-tumor](https://docs.ultralytics.com/datasets/detect/brain-tumor/) MRI dataset **through the WeightsLab training pipeline**.\n", + "\n", + "Instead of Ultralytics' one-shot `train`/`predict` flow, we hand training to `WLAwareTrainer`. It wraps the train/val dataloaders, logs a **per-sample** signal for every image (loss, prediction stats, tags) and ships live prediction overlays to **Weights Studio** — where you inspect the data grid, remove/weight samples, edit the model and resume, all while training runs.\n", + "\n", + "**Dataset** — 893 training images and 223 validation images, each with detection annotations. Classes: `0: negative`, `1: positive`. Ultralytics auto-downloads it (~4 MB) the first time you train." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o7YFKEN1Ks7Y" + }, + "source": [ + "## 1 · Setup\n", + "\n", + "Install WeightsLab (which pulls in Ultralytics) and verify the environment." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "71be394c", + "outputId": "c9ef4767-84f5-4bea-e395-9e11766002f2" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (75.2.0)\n", + "Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (0.47.0)\n", + "Requirement already satisfied: packaging>=24.0 in /usr/local/lib/python3.12/dist-packages (from wheel) (26.2)\n" + ] + } + ], + "source": [ + "%pip install setuptools wheel" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "jV1oi_PQN0xK", + "outputId": "eda8eca1-f4b3-4ffd-eaa6-0f67ce857153" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting git+https://github.com/GrayboxTech/weightslab.git@landingcollab\n", + " Cloning https://github.com/GrayboxTech/weightslab.git (to revision landingcollab) to /tmp/pip-req-build-z4ba7cx9\n", + " Running command git clone --filter=blob:none --quiet https://github.com/GrayboxTech/weightslab.git /tmp/pip-req-build-z4ba7cx9\n", + " Running command git checkout -b landingcollab --track origin/landingcollab\n", + " Switched to a new branch 'landingcollab'\n", + " Branch 'landingcollab' set up to track remote branch 'landingcollab' from 'origin'.\n", + " Resolved https://github.com/GrayboxTech/weightslab.git to commit 3f1cf143bc93e67df43adc71ab831fa71477e0d2\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.10.2)\n", + "Requirement already satisfied: torch<=2.9,>=2.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.9.0)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.24.0)\n", + "Requirement already satisfied: torchmetrics>=1.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.21)\n", + "Requirement already satisfied: onnx<=1.20,>=1.15 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.20.0)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.4.9)\n", + "Requirement already satisfied: langchain-ollama<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.1.0)\n", + "Requirement already satisfied: langchain-openai<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.5)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev8) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.2)\n", + "Requirement already satisfied: ollama<1.0.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev8) (0.6.2)\n", + "Requirement already satisfied: openai<3.0.0,>=2.45.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2.45.0)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev8) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.5.0)\n", + "Requirement already satisfied: lightning-utilities>=0.15.3 in /usr/local/lib/python3.12/dist-packages (from torchmetrics>=1.9->weightslab==1.3.3.dev8) (0.15.3)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.0.0)\n", + "Building wheels for collected packages: weightslab\n", + " Building wheel for weightslab (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for weightslab: filename=weightslab-1.3.3.dev8-py3-none-any.whl size=2827715 sha256=5c4e7c14bbf95a48b9a8c86b5a8c744af67e1eb395197cbe79d3e443c21cf86e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-__zwgf7x/wheels/4d/c7/cd/817c2745790555e7b6c532f5b6055d47567f9416fdfc65879b\n", + "Successfully built weightslab\n", + "Installing collected packages: weightslab\n", + " Attempting uninstall: weightslab\n", + " Found existing installation: weightslab 1.3.3.dev7\n", + " Uninstalling weightslab-1.3.3.dev7:\n", + " Successfully uninstalled weightslab-1.3.3.dev7\n", + "Successfully installed weightslab-1.3.3.dev8\n" + ] + }, + { + "data": { + "application/vnd.colab-display-data+json": { + "id": "b7fce274042740d19087972fe9c01c9b", + "pip_warning": { + "packages": [ + "weightslab" + ] + } + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%pip install git+https://github.com/GrayboxTech/weightslab.git" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pJhvREbaKs7f", + "outputId": "651baa36-9ebb-49da-830b-e6c8a6e2b64f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ultralytics 8.4.96 🚀 Python-3.12.13 torch-2.9.0+cu128 CPU (Intel Xeon CPU @ 2.20GHz)\n", + "Setup complete ✅ (2 CPUs, 12.7 GB RAM, 30.5/107.7 GB disk)\n", + "16/07/2026-09:29:29.683 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmp6meuzqdz/weightslab_logs/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:29.685 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-09:29:29.686 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev8 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-09:29:29.690 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install weightslab\n", + "# %pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev8\"\n", + "%pip install ultralytics\n", + "\n", + "import ultralytics\n", + "ultralytics.checks()\n", + "\n", + "import weightslab as wl" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QA3hHC2xKs7h" + }, + "source": [ + "## 2 · Dataset\n", + "\n", + "The dataset is described by a small YAML file that Ultralytics resolves and downloads automatically — no manual setup needed. For reference, `brain-tumor.yaml` looks like this:\n", + "\n", + "```yaml\n", + "path: brain-tumor # dataset root dir (auto-downloaded)\n", + "train: images/train # 893 images\n", + "val: images/val # 223 images\n", + "names:\n", + " 0: negative\n", + " 1: positive\n", + "```\n", + "\n", + "To train on **your own** data, point `data` (in the next cell) at your own `data.yaml` in YOLO format." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rH79t30YKs7i" + }, + "source": [ + "## 3 · Train through WeightsLab\n", + "\n", + "We register the run's hyperparameters with `wl.watch_or_edit(...)` (so they become live-editable from Studio), start the backend services with `wl.serve()`, then hand training to `WLAwareTrainer` via the standard `YOLO(...).train(trainer=...)` entry point.\n", + "\n", + "> **Connect Weights Studio** — run `weightslab ui launch` locally (opens `http://localhost:5173`) *before or during* training to watch the run live. On **Colab**, serve over a public tunnel instead: use `wl.serve(serving_grpc=True, serving_bore=True)` and connect your local Studio with `weightslab tunnel bore.pub:PORT` (the port is printed when serving starts).\n", + "\n", + "Data augmentations are disabled so each sample maps cleanly to its ground truth in the grid." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "Svz7NK9qKs7k", + "outputId": "2931e6b9-a06e-4dc0-9f67-e2e6216ee1c9" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16/07/2026-09:29:52.825 INFO:weightslab.src:watch_or_edit: LoggerQueue for experiment history has been initialized and registered.\n", + "16/07/2026-09:29:52.828 INFO:weightslab.components.checkpoint_manager:__init__: CheckpointManager initialized at /tmp/tmp3abnx3ay\n", + "16/07/2026-09:29:52.829 INFO:weightslab.src:watch_or_edit: Registered new checkpoint manager in ledger\n", + "16/07/2026-09:29:52.830 INFO:root:set_log_directory: Log file moved from /tmp/tmp6meuzqdz/weightslab_logs/weightslab_20260716_092929.log to /tmp/tmp3abnx3ay/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:52.832 INFO:root:set_log_directory: Log directory updated to: /tmp/tmp3abnx3ay\n", + "16/07/2026-09:29:52.833 INFO:root:set_log_directory: Log file: /tmp/tmp3abnx3ay/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:52.835 INFO:weightslab.trainer.trainer_services:_run_security_preflight: \n", + "\n", + "# #######################################\n", + "# #######################################\n", + "[gRPC] Security preflight checks:\n", + "\tTLS: DISABLED (unencrypted traffic)\n", + "\tAuth tokens: NONE configured\n", + "\t! WARNING: GRPC_TLS_ENABLED=0. Traffic will be unencrypted. Use only for development.\n", + "\t! WARNING: No GRPC_AUTH_TOKEN/GRPC_AUTH_TOKENS configured. Only transport-level trust (TLS/mTLS) will protect RPC access.\n", + "# #######################################\n", + "# #######################################\n", + "\n", + "16/07/2026-09:29:52.837 INFO:weightslab.trainer.trainer_services:grpc_serve: [gRPC] Watchdogs disabled via WEIGHTSLAB_DISABLE_WATCHDOGS.\n", + "16/07/2026-09:29:52.839 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Thread callback started\n", + "16/07/2026-09:29:52.840 INFO:weightslab.trainer.trainer_services:grpc_serve: [gRPC] Server started with watchdogs disabled (host=0.0.0.0 port=50051 workers=None)\n", + "16/07/2026-09:29:52.841 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Creating ThreadPoolExecutor with 6 worker threads (n_workers_grpc=None, max_concurrent_rpcs=None)\n", + "16/07/2026-09:29:52.842 WARNING:weightslab.src:serve: Running in a notebook/Colab with serving_grpc=True but serving_bore=False. Weights Studio runs on your OWN machine and cannot reach this backend directly. Pass serving_bore=True to open a tunnel and sync with the UI: wl.serve(serving_grpc=True, serving_bore=True).\n", + "16/07/2026-09:29:52.852 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Server object created\n", + "16/07/2026-09:29:52.854 INFO:weightslab.backend.cli:cli_serve: cli_thread_started\n", + "16/07/2026-09:29:52.859 INFO:weightslab.trainer.services.agent.agent:__init__: Initializing DataManipulationAgent\n", + "16/07/2026-09:29:52.860 INFO:weightslab.trainer.services.agent.agent:_setup_model_schema: [Agent] components.get('model') returned None (ledger-registered model names: []). Model-related agent requests will report 'no model registered'. If a model IS registered under a name other than the experiment name / 'experiment' / 'main', ExperimentContext.ensure_components()'s model-resolution heuristic won't find it.\n", + "16/07/2026-09:29:52.867 WARNING:weightslab.trainer.services.agent.agent:_load_config: Error loading config from /usr/local/lib/python3.12/dist-packages: [Errno 21] Is a directory: '/usr/local/lib/python3.12/dist-packages'\n", + "16/07/2026-09:29:52.869 INFO:weightslab.trainer.services.agent.agent:_load_config: \n", + "\n", + "# #######################################\n", + "# #######################################\n", + "Agent initialized from configuration /content/agent_config.yaml: \n", + "\tFinal Agent Configuration: Preferred Provider=openrouter, \n", + "\tFallback to Local=True, \n", + "\tOpenRouter Model=~google/gemini-flash-latest with:\n", + "\t\tAPI Key=None\n", + "\t\tBase URL=https://openrouter.ai/api/v1, \n", + "\tOllama Model=llama3.2:3b\n", + "# #######################################\n", + "# #######################################\n", + "\n", + "16/07/2026-09:29:52.870 INFO:weightslab.trainer.services.agent.agent:_setup_providers: Setting up Ollama with model llama3.2:3b\n", + "16/07/2026-09:29:52.915 INFO:weightslab.components.global_monitoring:resume: \n", + "Attempting to resume training...\n", + "16/07/2026-09:29:52.916 INFO:weightslab.components.checkpoint_manager:update_experiment_hash: First time initialization; skipping hash update.\n", + "16/07/2026-09:29:52.917 INFO:weightslab.components.experiment_hash:has_changed: Experiment configuration changed: {'hp'}\n", + "16/07/2026-09:29:52.918 INFO:weightslab.components.experiment_hash:generate_hash: Generated experiment hash: 1b111ac40000000000000000- (HP: 1b111ac4, Model: 00000000, Data: 00000000)\n", + "16/07/2026-09:29:52.919 INFO:weightslab.components.checkpoint_manager:update_experiment_hash: Initial experiment hash set: 1b111ac4-00000000-00000000\n", + "16/07/2026-09:29:52.919 INFO:weightslab.components.checkpoint_manager:update_experiment_hash: Changed components: {'hp'}\n", + "16/07/2026-09:29:52.922 INFO:weightslab.components.checkpoint_manager:_save_changes: Dumping hyperparameters config...\n", + "16/07/2026-09:29:52.924 INFO:weightslab.components.checkpoint_manager:save_config: Saved config: 1b111ac4_config.yaml with exp_hash prefix: 1b111ac4\n", + "16/07/2026-09:29:52.925 WARNING:weightslab.components.checkpoint_manager:_save_changes: Could not save weights: no model available\n", + "16/07/2026-09:29:52.931 INFO:weightslab.components.checkpoint_manager:save_logger_snapshot: Saved logger snapshot: /tmp/tmp3abnx3ay/checkpoints/loggers/loggers.manifest.json (1 chunks)\n", + "16/07/2026-09:29:52.935 INFO:weightslab.components.global_monitoring:resume: Hashes by module: ['1b111ac4', '00000000', '00000000']\n", + "16/07/2026-09:29:52.935 WARNING:weightslab.components.global_monitoring:resume: Cannot resume training: experiment hash not computed yet for every modules ['1b111ac4', '00000000', '00000000'].\n", + "16/07/2026-09:29:53.028 INFO:weightslab.trainer.services.agent.agent:_setup_providers: [Agent] Ollama enabled: llama3.2:3b\n", + "16/07/2026-09:29:53.031 INFO:weightslab.trainer.services.data_service:__init__: DataService initialized.\n", + "16/07/2026-09:29:53.036 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Servicer added\n", + "16/07/2026-09:29:53.040 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Attempting to bind to 0.0.0.0:50051\n", + "16/07/2026-09:29:53.044 WARNING:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] TLS disabled; using insecure transport on 0.0.0.0:50051\n", + "16/07/2026-09:29:53.046 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Port 50051 bound successfully.\n", + "16/07/2026-09:29:53.051 INFO:weightslab.trainer.trainer_services:serving_thread_callback: [gRPC] Server started and listening on 0.0.0.0:50051\n", + "Ultralytics 8.4.96 🚀 Python-3.12.13 torch-2.9.0+cu128 CPU (Intel Xeon CPU @ 2.20GHz)\n", + "\u001b[34m\u001b[1mengine/trainer: \u001b[0magnostic_nms=False, amp=False, angle=1.0, augment=False, auto_augment=None, batch=16, bgr=0.0, box=7.5, cache=False, cfg=None, classes=None, close_mosaic=10, cls=0.5, cls_pw=0.0, cls_remap=True, compile=False, conf=None, copy_paste=0.0, copy_paste_mode=flip, cos_lr=False, cutmix=0.0, data=/content/datasets/brain-tumor/brain-tumor.yaml, degrees=0.0, deterministic=True, device=cpu, dfl=1.5, dis=6.0, distill_model=None, dnn=False, dropout=0.0, dynamic=False, embed=None, end2end=None, epochs=10, erasing=0.0, exist_ok=False, fliplr=0.0, flipud=0.0, format=torchscript, fraction=1.0, freeze=None, hsv_h=0.0, hsv_s=0.0, hsv_v=0.0, imgsz=640, iou=0.7, keras=False, kobj=1.0, line_width=None, lr0=0.001, lrf=0.01, mask_ratio=4, max_det=300, mixup=0.0, mode=train, model=yolo11n.pt, momentum=0.937, mosaic=0.0, multi_scale=0.0, name=brain-tumor-4, nbs=64, nms=False, opset=None, optimize=False, optimizer=SGD, overlap_mask=True, patience=100, perspective=0.0, plots=True, pose=12.0, pretrained=True, profile=False, project=./wl_logs, quantize=None, rect=False, resume=False, retina_masks=False, rle=1.0, save=True, save_conf=False, save_crop=False, save_dir=/content/runs/detect/wl_logs/brain-tumor-4, save_frames=False, save_json=False, save_period=-1, save_txt=False, scale=0.0, seed=0, shear=0.0, show=False, show_boxes=True, show_conf=True, show_labels=True, simplify=True, single_cls=False, source=None, split=val, stream_buffer=False, task=detect, time=None, tracker=tracktrack.yaml, translate=0.0, val=True, verbose=True, vid_stride=1, visualize=False, warmup_bias_lr=0.1, warmup_epochs=3.0, warmup_momentum=0.8, weight_decay=0.0005, workers=0, workspace=None\n", + "Overriding model.yaml nc=80 with nc=2\n", + "\n", + " from n params module arguments \n", + " 0 -1 1 464 ultralytics.nn.modules.conv.Conv [3, 16, 3, 2] \n", + " 1 -1 1 4672 ultralytics.nn.modules.conv.Conv [16, 32, 3, 2] \n", + " 2 -1 1 6640 ultralytics.nn.modules.block.C3k2 [32, 64, 1, False, 0.25] \n", + " 3 -1 1 36992 ultralytics.nn.modules.conv.Conv [64, 64, 3, 2] \n", + " 4 -1 1 26080 ultralytics.nn.modules.block.C3k2 [64, 128, 1, False, 0.25] \n", + " 5 -1 1 147712 ultralytics.nn.modules.conv.Conv [128, 128, 3, 2] \n", + " 6 -1 1 87040 ultralytics.nn.modules.block.C3k2 [128, 128, 1, True] \n", + " 7 -1 1 295424 ultralytics.nn.modules.conv.Conv [128, 256, 3, 2] \n", + " 8 -1 1 346112 ultralytics.nn.modules.block.C3k2 [256, 256, 1, True] \n", + " 9 -1 1 164608 ultralytics.nn.modules.block.SPPF [256, 256, 5] \n", + " 10 -1 1 249728 ultralytics.nn.modules.block.C2PSA [256, 256, 1] \n", + " 11 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest'] \n", + " 12 [-1, 6] 1 0 ultralytics.nn.modules.conv.Concat [1] \n", + " 13 -1 1 111296 ultralytics.nn.modules.block.C3k2 [384, 128, 1, False] \n", + " 14 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest'] \n", + " 15 [-1, 4] 1 0 ultralytics.nn.modules.conv.Concat [1] \n", + " 16 -1 1 32096 ultralytics.nn.modules.block.C3k2 [256, 64, 1, False] \n", + " 17 -1 1 36992 ultralytics.nn.modules.conv.Conv [64, 64, 3, 2] \n", + " 18 [-1, 13] 1 0 ultralytics.nn.modules.conv.Concat [1] \n", + " 19 -1 1 86720 ultralytics.nn.modules.block.C3k2 [192, 128, 1, False] \n", + " 20 -1 1 147712 ultralytics.nn.modules.conv.Conv [128, 128, 3, 2] \n", + " 21 [-1, 10] 1 0 ultralytics.nn.modules.conv.Concat [1] \n", + " 22 -1 1 378880 ultralytics.nn.modules.block.C3k2 [384, 256, 1, True] \n", + " 23 [16, 19, 22] 1 431062 ultralytics.nn.modules.head.Detect [2, 16, None, [64, 128, 256]] \n", + "YOLO11n summary: 182 layers, 2,590,230 parameters, 2,590,214 gradients, 6.4 GFLOPs\n", + "\n", + "Transferred 448/499 items from pretrained weights\n", + "Freezing layer 'model.23.dfl.conv.weight'\n", + "\u001b[34m\u001b[1mtrain: \u001b[0mFast image access ✅ (ping: 0.0±0.0 ms, read: 115.3±57.6 MB/s, size: 3.6 KB)\n", + "\u001b[K\u001b[34m\u001b[1mtrain: \u001b[0mScanning /content/datasets/brain-tumor/labels/train.cache... 878 images, 15 backgrounds, 0 corrupt: 100% ━━━━━━━━━━━━ 893/893 101.2Mit/s 0.0s\n", + "\u001b[34m\u001b[1malbumentations: \u001b[0mBlur(p=0.01, blur_limit=(3, 7)), MedianBlur(p=0.01, blur_limit=(3, 7)), ToGray(p=0.01, method='weighted_average', num_output_channels=3), CLAHE(p=0.01, clip_limit=(1.0, 4.0), tile_grid_size=(8, 8))\n", + "16/07/2026-09:29:54.526 INFO:weightslab.data.data_samples_with_ops:__init__: [DataSampleTrackingWrapper] H5 persistence enabled at /tmp/tmp3abnx3ay/checkpoints/data/data.h5\n", + "16/07/2026-09:29:54.529 INFO:weightslab.data.data_samples_with_ops:__init__: Preloading sample statistics for PHYSICAL indices in split 'train_loader' with: preload_labels=True, preload_metadata=True...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Initializing ledger for split 'train_loader': 100%|██████████| 893/893 [00:00<00:00, 13970.17it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16/07/2026-09:29:54.600 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] Registering split 'train_loader' with 893 samples.\n", + "16/07/2026-09:29:54.615 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] After annotation expansion: 893 samples → 984 annotation rows.\n", + "16/07/2026-09:29:54.616 INFO:weightslab.data.h5_array_store:__init__: [H5ArrayStore] Initialized with cache limit: 2048MB\n", + "16/07/2026-09:29:54.636 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loading checkpoint 1b111ac400000000...\n", + "16/07/2026-09:29:54.637 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Target: HP=1b111ac4 MODEL=00000000 DATA=00000000\n", + "16/07/2026-09:29:54.638 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Current: HP=1b111ac4 MODEL=00000000 DATA=00000000\n", + "16/07/2026-09:29:54.639 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Model architecture unchanged, using current model\n", + "16/07/2026-09:29:54.639 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Config unchanged, using current config\n", + "16/07/2026-09:29:54.641 WARNING:weightslab.components.checkpoint_manager:load_checkpoint: [WARNING] Data snapshot file not found: /tmp/tmp3abnx3ay/checkpoints/data/00000000/00000000_data_snapshot.json\n", + "16/07/2026-09:29:54.641 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loaded components: set()\n", + "\u001b[34m\u001b[1mval: \u001b[0mFast image access ✅ (ping: 0.0±0.0 ms, read: 62.7±38.0 MB/s, size: 3.9 KB)\n", + "\u001b[K\u001b[34m\u001b[1mval: \u001b[0mScanning /content/datasets/brain-tumor/labels/val.cache... 223 images, 0 backgrounds, 0 corrupt: 100% ━━━━━━━━━━━━ 223/223 32.3Mit/s 0.0s\n", + "16/07/2026-09:29:54.660 INFO:weightslab.data.data_samples_with_ops:__init__: [DataSampleTrackingWrapper] H5 persistence enabled at /tmp/tmp3abnx3ay/checkpoints/data/data.h5\n", + "16/07/2026-09:29:54.662 INFO:weightslab.data.data_samples_with_ops:__init__: Preloading sample statistics for PHYSICAL indices in split 'val_loader' with: preload_labels=True, preload_metadata=True...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "Initializing ledger for split 'val_loader': 100%|██████████| 223/223 [00:00<00:00, 5362.24it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16/07/2026-09:29:54.708 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] Registering split 'val_loader' with 223 samples.\n", + "16/07/2026-09:29:54.714 INFO:weightslab.data.dataframe_manager:register_split: [LedgeredDataFrameManager] After annotation expansion: 223 samples → 257 annotation rows.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16/07/2026-09:29:54.734 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loading checkpoint 1b111ac400000000...\n", + "16/07/2026-09:29:54.736 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Target: HP=1b111ac4 MODEL=00000000 DATA=00000000\n", + "16/07/2026-09:29:54.739 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Current: HP=1b111ac4 MODEL=00000000 DATA=00000000\n", + "16/07/2026-09:29:54.740 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Model architecture unchanged, using current model\n", + "16/07/2026-09:29:54.742 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Config unchanged, using current config\n", + "16/07/2026-09:29:54.743 WARNING:weightslab.components.checkpoint_manager:load_checkpoint: [WARNING] Data snapshot file not found: /tmp/tmp3abnx3ay/checkpoints/data/00000000/00000000_data_snapshot.json\n", + "16/07/2026-09:29:54.746 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loaded components: set()\n", + "\u001b[34m\u001b[1moptimizer:\u001b[0m SGD(lr=0.001, momentum=0.937) with parameter groups 81 weight(decay=0.0), 88 weight(decay=0.0005), 87 bias(decay=0.0)\n", + "Plotting labels to /content/runs/detect/wl_logs/brain-tumor-4/labels.jpg... \n", + "16/07/2026-09:29:55.639 INFO:weightslab.backend.model_interface:__init__: Using checkpoint manager from ledger\n", + "16/07/2026-09:29:55.644 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loading checkpoint 1b111ac400000000...\n", + "16/07/2026-09:29:55.645 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Target: HP=1b111ac4 MODEL=00000000 DATA=00000000\n", + "16/07/2026-09:29:55.645 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Current: HP=1b111ac4 MODEL=00000000 DATA=00000000\n", + "16/07/2026-09:29:55.646 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Model architecture unchanged, using current model\n", + "16/07/2026-09:29:55.647 WARNING:weightslab.components.checkpoint_manager:load_checkpoint: [WARNING] No weight files found for 00000000\n", + "16/07/2026-09:29:55.648 INFO:weightslab.components.checkpoint_manager:load_checkpoint: [-] Config unchanged, using current config\n", + "16/07/2026-09:29:55.648 INFO:weightslab.components.checkpoint_manager:load_checkpoint: Loaded components: set()\n", + "Image sizes 640 train, 640 val\n", + "Using 0 dataloader workers\n", + "Logging results to \u001b[1m/content/runs/detect/wl_logs/brain-tumor-4\u001b[0m\n", + "Starting training for 10 epochs...\n", + "16/07/2026-09:29:55.661 INFO:weightslab.backend.dataloader_interface:set_batch_size: Batch size updated: 16 -> 4 (Loader: train_loader)\n", + "Closing dataloader mosaic\n", + "\u001b[34m\u001b[1malbumentations: \u001b[0mBlur(p=0.01, blur_limit=(3, 7)), MedianBlur(p=0.01, blur_limit=(3, 7)), ToGray(p=0.01, method='weighted_average', num_output_channels=3), CLAHE(p=0.01, clip_limit=(1.0, 4.0), tile_grid_size=(8, 8))\n", + "\n", + " Epoch GPU_mem box_loss cls_loss dfl_loss Instances Size\n", + "\u001b[K: 0% ──────────── 0/56 1:03\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_6805/1732828701.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;31m# Read back the (now live) hyperparameters and hand training to WLAwareTrainer.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mYOLO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"model\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m results = model.train(\n\u001b[0m\u001b[1;32m 33\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mWLAwareTrainer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"data\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/ultralytics/engine/model.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, trainer, **kwargs)\u001b[0m\n\u001b[1;32m 814\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 815\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 816\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 817\u001b[0m \u001b[0;31m# Update model and cfg after training\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 818\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mRANK\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/ultralytics/engine/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 239\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 240\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 241\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 242\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 243\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_setup_scheduler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/ultralytics/engine/trainer.py\u001b[0m in \u001b[0;36m_do_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 430\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 431\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpbar\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 432\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"on_train_batch_start\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 433\u001b[0m \u001b[0;31m# Warmup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 434\u001b[0m \u001b[0mni\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnb\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/ultralytics/engine/trainer.py\u001b[0m in \u001b[0;36mrun_callbacks\u001b[0;34m(self, event)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[0;34m\"\"\"Run all existing callbacks associated with a particular event.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 212\u001b[0;31m \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 213\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/weightslab/integrations/ultralytics/trainer.py\u001b[0m in \u001b[0;36m_on_train_batch_start\u001b[0;34m(trainer)\u001b[0m\n\u001b[1;32m 97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_on_train_batch_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m \u001b[0mwl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mguard_training_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_on_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/weightslab/components/global_monitoring.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self, f)\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 225\u001b[0m \u001b[0mpause_controller\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresume\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mforce\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 226\u001b[0;31m \u001b[0mpause_controller\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_if_paused\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 227\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marchitecture_guard\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.12/dist-packages/weightslab/components/global_monitoring.py\u001b[0m in \u001b[0;36mwait_if_paused\u001b[0;34m(self, skip_pause)\u001b[0m\n\u001b[1;32m 119\u001b[0m \u001b[0;31m# Also wakes up early when an evaluation is pending/running so the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[0;31m# training loop and dataloaders can service evaluation mode.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 122\u001b[0m \u001b[0;31m# Timeout occurred – check for evaluation request before looping\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 123\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.12/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 653\u001b[0m \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 654\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 655\u001b[0;31m \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 656\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 657\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.12/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 357\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 358\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 359\u001b[0;31m \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 360\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 361\u001b[0m \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "import os\n", + "os.environ.setdefault(\"WL_PRELOAD_IMAGE_OVERVIEW\", \"0\")\n", + "os.environ.setdefault(\"WEIGHTSLAB_LOG_LEVEL\", \"WARNING\")\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "from weightslab.integrations.ultralytics import WLAwareTrainer\n", + "from ultralytics import YOLO\n", + "\n", + "# Hyperparameters registered with WeightsLab -> live-editable from Weights Studio.\n", + "cfg = {\n", + " \"model\": \"yolo11n.pt\", # pretrained checkpoint\n", + " \"data\": \"/content/datasets/brain-tumor/brain-tumor.yaml\", # Ultralytics auto-downloads (~4 MB)\n", + " \"image_size\": 640,\n", + " \"epochs\": 10,\n", + " # Per-sample prediction overlays streamed to Studio (NMS on the train set).\n", + " \"signals_cfg\": {\n", + " \"train_nms\": {\"conf_thres\": 0.25, \"iou_thres\": 0.45, \"max_nms\": 7},\n", + " },\n", + "}\n", + "wl.watch_or_edit(cfg, flag=\"hyperparameters\", defaults=cfg, poll_interval=1.0)\n", + "\n", + "# Start the WeightsLab backend services that Weights Studio connects to.\n", + "wl.serve(serving_grpc=True)\n", + "# Colab -> local Studio: wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "wl.start_training() # equivalent to pressing \"play\" in Studio\n", + "\n", + "# Read back the (now live) hyperparameters and hand training to WLAwareTrainer.\n", + "model = YOLO(cfg[\"model\"])\n", + "results = model.train(\n", + " trainer=WLAwareTrainer,\n", + " data=str(cfg[\"data\"]),\n", + " imgsz=cfg[\"image_size\"],\n", + " epochs=int(cfg[\"epochs\"]),\n", + " project=\"./wl_logs\", name=\"brain-tumor\", # -> WL log_dir/name\n", + " workers=0, # WL invariant (parent-process uid counter)\n", + " optimizer=\"SGD\", lr0=0.001, amp=False,\n", + " device='cpu',\n", + " # All augmentations off for a clean sample <-> ground-truth association.\n", + " mosaic=0.0, mixup=0.0, copy_paste=0.0,\n", + " hsv_h=0.0, hsv_s=0.0, hsv_v=0.0,\n", + " degrees=0.0, translate=0.0, scale=0.0, shear=0.0, perspective=0.0,\n", + " flipud=0.0, fliplr=0.0, erasing=0.0, auto_augment=None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f-Uw_s_8Ks7m" + }, + "source": [ + "## 4 · Explore the data grid\n", + "\n", + "The table below is exactly the data behind Weights Studio's **grid explorer**: one row per sample, keyed by split (`origin`) and sample id, with the per-sample loss, prediction stats and tags accumulated during training. This is the ledger that Studio reads from — here we render it inline with pandas.\n", + "\n", + "The image thumbnails and bounding-box overlays themselves are rendered by **Weights Studio** (which streams full previews over gRPC); the link below opens the interactive grid there." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "collapsed": true, + "id": "uOJdXGA3Ks7o", + "outputId": "26652fa3-6a15-43ce-9d75-034eaba21ccd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1241 samples tracked\n" + ] + }, + { + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "repr_error": "Out of range float values are not JSON compliant: nan", + "type": "dataframe" + }, + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
predictionprediction_rawtargetdiscardedorigintask_typelast_seengroup_idmember_rankimg_pathcls
sample_idannotation_id
00NoneNone[0.2335685, 0.254695, 0.4553995, 0.43075103, 1...Falsetrain_loader-1.000.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
10NoneNone[0.251174, 0.24061048, 0.44366202, 0.4307515, ...Falsetrain_loader-1.010.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
20NoneNoneNoneFalsetrain_loader-1.020.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.523474, 0.3978875, 0.634976, 0.48122054, 1....NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.40610352, 0.415493, 0.5234745, 0.522301, 1....NoneNaNNoneNaNNoneNaNNoneNone
30NoneNone[0.403756, 0.3826295, 0.637324, 0.5152585, 1.0...Falsetrain_loader-1.030.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
40NoneNone[0.4436615, 0.3814555, 0.5938965, 0.4518785, 1...Falsetrain_loader-1.040.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
50NoneNone[0.6044605, 0.3990615, 0.67253554, 0.46596247,...Falsetrain_loader-1.050.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
60NoneNone[0.410798, 0.4436625, 0.556338, 0.51173747, 1....Falsetrain_loader-1.060.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
70NoneNone[0.650235, 0.4166665, 0.73826295, 0.48356748, ...Falsetrain_loader-1.070.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
80NoneNone[0.64788747, 0.388498, 0.7582165, 0.49413198, ...Falsetrain_loader-1.080.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
90NoneNone[0.6807515, 0.40140802, 0.75704247, 0.45774597...Falsetrain_loader-1.090.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
100NoneNone[0.431925, 0.1701875, 0.561033, 0.3180745, 1.0...Falsetrain_loader-1.0100.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
110NoneNone[0.42018852, 0.34507048, 0.5434275, 0.4941315,...Falsetrain_loader-1.0110.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
120NoneNone[0.3732395, 0.348592, 0.54812247, 0.46009403, ...Falsetrain_loader-1.0120.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
130NoneNone[0.37793398, 0.3814555, 0.48122, 0.45892054, 1...Falsetrain_loader-1.0130.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
140NoneNoneNoneFalsetrain_loader-1.0140.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.43427247, 0.3556335, 0.51173747, 0.43075046...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4929575, 0.44718304, 0.5375585, 0.485915, 1...NoneNaNNoneNaNNoneNaNNoneNone
150NoneNoneNoneFalsetrain_loader-1.0150.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.629108, 0.39788702, 0.67840403, 0.453051, 1...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4530515, 0.4107985, 0.58568054, 0.5105635, ...NoneNaNNoneNaNNoneNaNNoneNone
160NoneNoneNoneFalsetrain_loader-1.0160.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.45070451, 0.3967135, 0.5762915, 0.5152585, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.63732404, 0.403756, 0.684272, 0.46009403, 1...NoneNaNNoneNaNNoneNaNNoneNone
170NoneNone[0.43779355, 0.3826295, 0.61032856, 0.5223005,...Falsetrain_loader-1.0170.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
180NoneNone[0.4401405, 0.374413, 0.6068075, 0.529343, 1.0...Falsetrain_loader-1.0180.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
190NoneNone[0.536385, 0.4495305, 0.600939, 0.49999946, 0....Falsetrain_loader-1.0190.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
200NoneNoneNoneFalsetrain_loader-1.0200.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0], [0.0]]
1NoneNone[0.52347445, 0.4424885, 0.6150235, 0.5281695, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.456573, 0.515258, 0.504695, 0.562206, 0.0, ...NoneNaNNoneNaNNoneNaNNoneNone
3NoneNone[0.504695, 0.44248852, 0.54342705, 0.48004752,...NoneNaNNoneNaNNoneNaNNoneNone
210NoneNone[0.54107946, 0.456573, 0.6103285, 0.511737, 0....Falsetrain_loader-1.0210.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
220NoneNone[0.3298125, 0.2042255, 0.4225355, 0.2957745, 0...Falsetrain_loader-1.0220.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
230NoneNone[0.517606, 0.18661949, 0.615024, 0.2875585, 1....Falsetrain_loader-1.0230.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
240NoneNone[0.51173747, 0.18779299, 0.6126765, 0.308685, ...Falsetrain_loader-1.0240.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
250NoneNoneNoneFalsetrain_loader-1.0250.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0]]
1NoneNone[0.30985948, 0.3626765, 0.3861505, 0.43896753,...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.41314548, 0.3427225, 0.48122054, 0.42018753...NoneNaNNoneNaNNoneNaNNoneNone
260NoneNone[0.25821602, 0.30046952, 0.46830997, 0.4577465...Falsetrain_loader-1.0260.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
270NoneNone[0.2734745, 0.2828635, 0.44718352, 0.4577465, ...Falsetrain_loader-1.0270.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
280NoneNone[0.5915495, 0.23591551, 0.6455405, 0.2887325, ...Falsetrain_loader-1.0280.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
290NoneNone[0.5903755, 0.231221, 0.6420185, 0.280517, 1.0...Falsetrain_loader-1.0290.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
300NoneNone[0.40493003, 0.2018775, 0.469484, 0.2699525, 1...Falsetrain_loader-1.0300.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
310NoneNone[0.39554, 0.19483551, 0.483568, 0.2887325, 1.0...Falsetrain_loader-1.0310.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
320NoneNone[0.4025825, 0.212441, 0.45539945, 0.269953, 1....Falsetrain_loader-1.0320.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
330NoneNone[0.2265255, 0.2136145, 0.3438965, 0.30751148, ...Falsetrain_loader-1.0330.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
340NoneNone[0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0]Falsetrain_loader-1.0340.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
350NoneNone[0.6079815, 0.320423, 0.7969485, 0.489437, 1.0...Falsetrain_loader-1.0350.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
360NoneNone[0.6185445, 0.30164298, 0.7758215, 0.494131, 1...Falsetrain_loader-1.0360.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "text/plain": [ + " prediction prediction_raw \\\n", + "sample_id annotation_id \n", + "0 0 None None \n", + "1 0 None None \n", + "2 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "3 0 None None \n", + "4 0 None None \n", + "5 0 None None \n", + "6 0 None None \n", + "7 0 None None \n", + "8 0 None None \n", + "9 0 None None \n", + "10 0 None None \n", + "11 0 None None \n", + "12 0 None None \n", + "13 0 None None \n", + "14 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "15 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "16 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "17 0 None None \n", + "18 0 None None \n", + "19 0 None None \n", + "20 0 None None \n", + " 1 None None \n", + " 2 None None \n", + " 3 None None \n", + "21 0 None None \n", + "22 0 None None \n", + "23 0 None None \n", + "24 0 None None \n", + "25 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "26 0 None None \n", + "27 0 None None \n", + "28 0 None None \n", + "29 0 None None \n", + "30 0 None None \n", + "31 0 None None \n", + "32 0 None None \n", + "33 0 None None \n", + "34 0 None None \n", + "35 0 None None \n", + "36 0 None None \n", + "\n", + " target \\\n", + "sample_id annotation_id \n", + "0 0 [0.2335685, 0.254695, 0.4553995, 0.43075103, 1... \n", + "1 0 [0.251174, 0.24061048, 0.44366202, 0.4307515, ... \n", + "2 0 None \n", + " 1 [0.523474, 0.3978875, 0.634976, 0.48122054, 1.... \n", + " 2 [0.40610352, 0.415493, 0.5234745, 0.522301, 1.... \n", + "3 0 [0.403756, 0.3826295, 0.637324, 0.5152585, 1.0... \n", + "4 0 [0.4436615, 0.3814555, 0.5938965, 0.4518785, 1... \n", + "5 0 [0.6044605, 0.3990615, 0.67253554, 0.46596247,... \n", + "6 0 [0.410798, 0.4436625, 0.556338, 0.51173747, 1.... \n", + "7 0 [0.650235, 0.4166665, 0.73826295, 0.48356748, ... \n", + "8 0 [0.64788747, 0.388498, 0.7582165, 0.49413198, ... \n", + "9 0 [0.6807515, 0.40140802, 0.75704247, 0.45774597... \n", + "10 0 [0.431925, 0.1701875, 0.561033, 0.3180745, 1.0... \n", + "11 0 [0.42018852, 0.34507048, 0.5434275, 0.4941315,... \n", + "12 0 [0.3732395, 0.348592, 0.54812247, 0.46009403, ... \n", + "13 0 [0.37793398, 0.3814555, 0.48122, 0.45892054, 1... \n", + "14 0 None \n", + " 1 [0.43427247, 0.3556335, 0.51173747, 0.43075046... \n", + " 2 [0.4929575, 0.44718304, 0.5375585, 0.485915, 1... \n", + "15 0 None \n", + " 1 [0.629108, 0.39788702, 0.67840403, 0.453051, 1... \n", + " 2 [0.4530515, 0.4107985, 0.58568054, 0.5105635, ... \n", + "16 0 None \n", + " 1 [0.45070451, 0.3967135, 0.5762915, 0.5152585, ... \n", + " 2 [0.63732404, 0.403756, 0.684272, 0.46009403, 1... \n", + "17 0 [0.43779355, 0.3826295, 0.61032856, 0.5223005,... \n", + "18 0 [0.4401405, 0.374413, 0.6068075, 0.529343, 1.0... \n", + "19 0 [0.536385, 0.4495305, 0.600939, 0.49999946, 0.... \n", + "20 0 None \n", + " 1 [0.52347445, 0.4424885, 0.6150235, 0.5281695, ... \n", + " 2 [0.456573, 0.515258, 0.504695, 0.562206, 0.0, ... \n", + " 3 [0.504695, 0.44248852, 0.54342705, 0.48004752,... \n", + "21 0 [0.54107946, 0.456573, 0.6103285, 0.511737, 0.... \n", + "22 0 [0.3298125, 0.2042255, 0.4225355, 0.2957745, 0... \n", + "23 0 [0.517606, 0.18661949, 0.615024, 0.2875585, 1.... \n", + "24 0 [0.51173747, 0.18779299, 0.6126765, 0.308685, ... \n", + "25 0 None \n", + " 1 [0.30985948, 0.3626765, 0.3861505, 0.43896753,... \n", + " 2 [0.41314548, 0.3427225, 0.48122054, 0.42018753... \n", + "26 0 [0.25821602, 0.30046952, 0.46830997, 0.4577465... \n", + "27 0 [0.2734745, 0.2828635, 0.44718352, 0.4577465, ... \n", + "28 0 [0.5915495, 0.23591551, 0.6455405, 0.2887325, ... \n", + "29 0 [0.5903755, 0.231221, 0.6420185, 0.280517, 1.0... \n", + "30 0 [0.40493003, 0.2018775, 0.469484, 0.2699525, 1... \n", + "31 0 [0.39554, 0.19483551, 0.483568, 0.2887325, 1.0... \n", + "32 0 [0.4025825, 0.212441, 0.45539945, 0.269953, 1.... \n", + "33 0 [0.2265255, 0.2136145, 0.3438965, 0.30751148, ... \n", + "34 0 [0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0] \n", + "35 0 [0.6079815, 0.320423, 0.7969485, 0.489437, 1.0... \n", + "36 0 [0.6185445, 0.30164298, 0.7758215, 0.494131, 1... \n", + "\n", + " discarded origin task_type last_seen group_id \\\n", + "sample_id annotation_id \n", + "0 0 False train_loader -1.0 0 \n", + "1 0 False train_loader -1.0 1 \n", + "2 0 False train_loader -1.0 2 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "3 0 False train_loader -1.0 3 \n", + "4 0 False train_loader -1.0 4 \n", + "5 0 False train_loader -1.0 5 \n", + "6 0 False train_loader -1.0 6 \n", + "7 0 False train_loader -1.0 7 \n", + "8 0 False train_loader -1.0 8 \n", + "9 0 False train_loader -1.0 9 \n", + "10 0 False train_loader -1.0 10 \n", + "11 0 False train_loader -1.0 11 \n", + "12 0 False train_loader -1.0 12 \n", + "13 0 False train_loader -1.0 13 \n", + "14 0 False train_loader -1.0 14 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "15 0 False train_loader -1.0 15 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "16 0 False train_loader -1.0 16 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "17 0 False train_loader -1.0 17 \n", + "18 0 False train_loader -1.0 18 \n", + "19 0 False train_loader -1.0 19 \n", + "20 0 False train_loader -1.0 20 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + " 3 None NaN None NaN None \n", + "21 0 False train_loader -1.0 21 \n", + "22 0 False train_loader -1.0 22 \n", + "23 0 False train_loader -1.0 23 \n", + "24 0 False train_loader -1.0 24 \n", + "25 0 False train_loader -1.0 25 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "26 0 False train_loader -1.0 26 \n", + "27 0 False train_loader -1.0 27 \n", + "28 0 False train_loader -1.0 28 \n", + "29 0 False train_loader -1.0 29 \n", + "30 0 False train_loader -1.0 30 \n", + "31 0 False train_loader -1.0 31 \n", + "32 0 False train_loader -1.0 32 \n", + "33 0 False train_loader -1.0 33 \n", + "34 0 False train_loader -1.0 34 \n", + "35 0 False train_loader -1.0 35 \n", + "36 0 False train_loader -1.0 36 \n", + "\n", + " member_rank \\\n", + "sample_id annotation_id \n", + "0 0 0.0 \n", + "1 0 0.0 \n", + "2 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "3 0 0.0 \n", + "4 0 0.0 \n", + "5 0 0.0 \n", + "6 0 0.0 \n", + "7 0 0.0 \n", + "8 0 0.0 \n", + "9 0 0.0 \n", + "10 0 0.0 \n", + "11 0 0.0 \n", + "12 0 0.0 \n", + "13 0 0.0 \n", + "14 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "15 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "16 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "17 0 0.0 \n", + "18 0 0.0 \n", + "19 0 0.0 \n", + "20 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + " 3 NaN \n", + "21 0 0.0 \n", + "22 0 0.0 \n", + "23 0 0.0 \n", + "24 0 0.0 \n", + "25 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "26 0 0.0 \n", + "27 0 0.0 \n", + "28 0 0.0 \n", + "29 0 0.0 \n", + "30 0 0.0 \n", + "31 0 0.0 \n", + "32 0 0.0 \n", + "33 0 0.0 \n", + "34 0 0.0 \n", + "35 0 0.0 \n", + "36 0 0.0 \n", + "\n", + " img_path \\\n", + "sample_id annotation_id \n", + "0 0 /content/datasets/brain-tumor/images/train/000... \n", + "1 0 /content/datasets/brain-tumor/images/train/000... \n", + "2 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "3 0 /content/datasets/brain-tumor/images/train/000... \n", + "4 0 /content/datasets/brain-tumor/images/train/000... \n", + "5 0 /content/datasets/brain-tumor/images/train/000... \n", + "6 0 /content/datasets/brain-tumor/images/train/000... \n", + "7 0 /content/datasets/brain-tumor/images/train/000... \n", + "8 0 /content/datasets/brain-tumor/images/train/000... \n", + "9 0 /content/datasets/brain-tumor/images/train/000... \n", + "10 0 /content/datasets/brain-tumor/images/train/000... \n", + "11 0 /content/datasets/brain-tumor/images/train/000... \n", + "12 0 /content/datasets/brain-tumor/images/train/000... \n", + "13 0 /content/datasets/brain-tumor/images/train/000... \n", + "14 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "15 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "16 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "17 0 /content/datasets/brain-tumor/images/train/000... \n", + "18 0 /content/datasets/brain-tumor/images/train/000... \n", + "19 0 /content/datasets/brain-tumor/images/train/000... \n", + "20 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 /content/datasets/brain-tumor/images/train/000... \n", + "22 0 /content/datasets/brain-tumor/images/train/000... \n", + "23 0 /content/datasets/brain-tumor/images/train/000... \n", + "24 0 /content/datasets/brain-tumor/images/train/000... \n", + "25 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "26 0 /content/datasets/brain-tumor/images/train/000... \n", + "27 0 /content/datasets/brain-tumor/images/train/000... \n", + "28 0 /content/datasets/brain-tumor/images/train/000... \n", + "29 0 /content/datasets/brain-tumor/images/train/000... \n", + "30 0 /content/datasets/brain-tumor/images/train/000... \n", + "31 0 /content/datasets/brain-tumor/images/train/000... \n", + "32 0 /content/datasets/brain-tumor/images/train/000... \n", + "33 0 /content/datasets/brain-tumor/images/train/000... \n", + "34 0 /content/datasets/brain-tumor/images/train/000... \n", + "35 0 /content/datasets/brain-tumor/images/train/000... \n", + "36 0 /content/datasets/brain-tumor/images/train/000... \n", + "\n", + " cls \n", + "sample_id annotation_id \n", + "0 0 [[1.0]] \n", + "1 0 [[1.0]] \n", + "2 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "3 0 [[1.0]] \n", + "4 0 [[1.0]] \n", + "5 0 [[1.0]] \n", + "6 0 [[1.0]] \n", + "7 0 [[1.0]] \n", + "8 0 [[1.0]] \n", + "9 0 [[1.0]] \n", + "10 0 [[1.0]] \n", + "11 0 [[1.0]] \n", + "12 0 [[1.0]] \n", + "13 0 [[1.0]] \n", + "14 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "15 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "16 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "17 0 [[1.0]] \n", + "18 0 [[1.0]] \n", + "19 0 [[0.0]] \n", + "20 0 [[0.0], [0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 [[0.0]] \n", + "22 0 [[0.0]] \n", + "23 0 [[1.0]] \n", + "24 0 [[1.0]] \n", + "25 0 [[0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + "26 0 [[0.0]] \n", + "27 0 [[0.0]] \n", + "28 0 [[1.0]] \n", + "29 0 [[1.0]] \n", + "30 0 [[1.0]] \n", + "31 0 [[1.0]] \n", + "32 0 [[1.0]] \n", + "33 0 [[1.0]] \n", + "34 0 [[1.0]] \n", + "35 0 [[1.0]] \n", + "36 0 [[1.0]] " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "🔗 Open the full grid in Weights Studio" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "from weightslab.backend.ledgers import get_dataframe\n", + "\n", + "# `get_df_view()` is the same per-sample DataFrame that powers Studio's grid.\n", + "dfm = get_dataframe()\n", + "grid = dfm.get_df_view() if hasattr(dfm, \"get_df_view\") else pd.DataFrame()\n", + "print(f\"{len(grid)} samples tracked\")\n", + "display(grid.head(50))\n", + "\n", + "# Open the live, interactive grid (with image + bbox previews) in Weights Studio.\n", + "# Studio must be running locally (`weightslab start`).\n", + "display(HTML(\n", + " '🔗 Open the full grid in Weights Studio'\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FXQ0gBOSKs7t" + }, + "source": [ + "> **Can clicking a grid row here jump to that sample in Studio?** Not out of the box today. The notebook and Studio are separate processes, and the link above opens Studio at its root. Per-sample deep-linking (e.g. `http://localhost:5173/?sample=` selecting and scrolling to that row) is a small frontend addition to Weights Studio — the sample `uid` is already the grid's index here, so the notebook side is ready for it once Studio accepts the query param." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iCI7FL86Ks7v" + }, + "source": [ + "## Ultralytics Citation\n", + "\n", + "```bibtex\n", + "@dataset{Jocher_Ultralytics_Datasets_2024,\n", + " author = {Jocher, Glenn and Rizwan, Muhammad},\n", + " license = {AGPL-3.0},\n", + " title = {Ultralytics Datasets: Brain-tumor Detection Dataset},\n", + " url = {https://docs.ultralytics.com/datasets/detect/brain-tumor/},\n", + " version = {1.0.0},\n", + " year = {2024}\n", + "}\n", + "```" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-carparts-segmentation-dataset.ipynb b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-carparts-segmentation-dataset.ipynb new file mode 100644 index 00000000..152db5b5 --- /dev/null +++ b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-carparts-segmentation-dataset.ipynb @@ -0,0 +1,1704 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Train an Ultralytics YOLO segmentation model on the carparts-seg dataset and stream every sample, prediction and metric live into Weights Studio to inspect, edit and evolve your run.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Carparts Segmentation · Ultralytics YOLO + WeightsLab\n", + "\n", + "This notebook trains a YOLO segmentation model on the [carparts-seg](https://docs.ultralytics.com/) dataset (segmenting individual car body parts) **through the WeightsLab training pipeline**.\n", + "\n", + "Instead of Ultralytics' one-shot `train`/`predict` flow, we hand training to `WLAwareSegmentationTrainer`. It wraps the train/val dataloaders, logs a **per-sample** signal for every image (loss, prediction stats, tags) and ships live prediction overlays to **Weights Studio** — where you inspect the data grid, remove/weight samples, edit the model and resume, all while training runs.\n", + "\n", + "Ultralytics auto-downloads the dataset the first time you train." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o7YFKEN1Ks7Y" + }, + "source": [ + "## 1 · Setup\n", + "\n", + "Install WeightsLab (which pulls in Ultralytics) and verify the environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "71be394c", + "outputId": "c9ef4767-84f5-4bea-e395-9e11766002f2", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "source": [ + "%pip install setuptools wheel" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (75.2.0)\n", + "Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (0.47.0)\n", + "Requirement already satisfied: packaging>=24.0 in /usr/local/lib/python3.12/dist-packages (from wheel) (26.2)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "%pip install git+https://github.com/GrayboxTech/weightslab.git@landingcollab" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "jV1oi_PQN0xK", + "outputId": "eda8eca1-f4b3-4ffd-eaa6-0f67ce857153" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting git+https://github.com/GrayboxTech/weightslab.git@landingcollab\n", + " Cloning https://github.com/GrayboxTech/weightslab.git (to revision landingcollab) to /tmp/pip-req-build-z4ba7cx9\n", + " Running command git clone --filter=blob:none --quiet https://github.com/GrayboxTech/weightslab.git /tmp/pip-req-build-z4ba7cx9\n", + " Running command git checkout -b landingcollab --track origin/landingcollab\n", + " Switched to a new branch 'landingcollab'\n", + " Branch 'landingcollab' set up to track remote branch 'landingcollab' from 'origin'.\n", + " Resolved https://github.com/GrayboxTech/weightslab.git to commit 3f1cf143bc93e67df43adc71ab831fa71477e0d2\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.10.2)\n", + "Requirement already satisfied: torch<=2.9,>=2.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.9.0)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.24.0)\n", + "Requirement already satisfied: torchmetrics>=1.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.21)\n", + "Requirement already satisfied: onnx<=1.20,>=1.15 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.20.0)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.4.9)\n", + "Requirement already satisfied: langchain-ollama<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.1.0)\n", + "Requirement already satisfied: langchain-openai<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.5)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev8) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.2)\n", + "Requirement already satisfied: ollama<1.0.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev8) (0.6.2)\n", + "Requirement already satisfied: openai<3.0.0,>=2.45.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2.45.0)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev8) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.5.0)\n", + "Requirement already satisfied: lightning-utilities>=0.15.3 in /usr/local/lib/python3.12/dist-packages (from torchmetrics>=1.9->weightslab==1.3.3.dev8) (0.15.3)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.0.0)\n", + "Building wheels for collected packages: weightslab\n", + " Building wheel for weightslab (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for weightslab: filename=weightslab-1.3.3.dev8-py3-none-any.whl size=2827715 sha256=5c4e7c14bbf95a48b9a8c86b5a8c744af67e1eb395197cbe79d3e443c21cf86e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-__zwgf7x/wheels/4d/c7/cd/817c2745790555e7b6c532f5b6055d47567f9416fdfc65879b\n", + "Successfully built weightslab\n", + "Installing collected packages: weightslab\n", + " Attempting uninstall: weightslab\n", + " Found existing installation: weightslab 1.3.3.dev7\n", + " Uninstalling weightslab-1.3.3.dev7:\n", + " Successfully uninstalled weightslab-1.3.3.dev7\n", + "Successfully installed weightslab-1.3.3.dev8\n" + ] + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "weightslab" + ] + }, + "id": "b7fce274042740d19087972fe9c01c9b" + } + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pJhvREbaKs7f", + "outputId": "651baa36-9ebb-49da-830b-e6c8a6e2b64f" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Ultralytics 8.4.96 🚀 Python-3.12.13 torch-2.9.0+cu128 CPU (Intel Xeon CPU @ 2.20GHz)\n", + "Setup complete ✅ (2 CPUs, 12.7 GB RAM, 30.5/107.7 GB disk)\n", + "16/07/2026-09:29:29.683 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmp6meuzqdz/weightslab_logs/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:29.685 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-09:29:29.686 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev8 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-09:29:29.690 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install weightslab\n", + "# %pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev8\"\n", + "%pip install ultralytics\n", + "\n", + "import ultralytics\n", + "ultralytics.checks()\n", + "\n", + "import weightslab as wl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 · Dataset\n", + "\n", + "The dataset is described by a small YAML file that Ultralytics resolves and downloads automatically. For reference:" + ] + }, + { + "cell_type": "markdown", + "source": [ + "```yaml\n", + "# Ultralytics YOLO 🚀, AGPL-3.0 license\n", + "# Carparts-seg dataset by Ultralytics\n", + "# Documentation: https://docs.ultralytics.com/datasets/segment/carparts-seg/\n", + "# Example usage: yolo train data=carparts-seg.yaml\n", + "# parent\n", + "# ├── ultralytics\n", + "# └── datasets\n", + "# └── carparts-seg ← downloads here (133 MB)\n", + "\n", + "# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..]\n", + "path: carparts-seg # dataset root dir\n", + "train: images/train # train images (relative to 'path') 3516 images\n", + "val: images/val # val images (relative to 'path') 276 images\n", + "test: images/test # test images (relative to 'path') 401 images\n", + "\n", + "# Classes\n", + "names:\n", + " 0: back_bumper\n", + " 1: back_door\n", + " 2: back_glass\n", + " 3: back_left_door\n", + " 4: back_left_light\n", + " 5: back_light\n", + " 6: back_right_door\n", + " 7: back_right_light\n", + " 8: front_bumper\n", + " 9: front_door\n", + " 10: front_glass\n", + " 11: front_left_door\n", + " 12: front_left_light\n", + " 13: front_light\n", + " 14: front_right_door\n", + " 15: front_right_light\n", + " 16: hood\n", + " 17: left_mirror\n", + " 18: object\n", + " 19: right_mirror\n", + " 20: tailgate\n", + " 21: trunk\n", + " 22: wheel\n", + "\n", + "# Download script/URL (optional)\n", + "download: https://github.com/ultralytics/assets/releases/download/v0.0.0/carparts-seg.zip\n", + "```" + ], + "metadata": { + "id": "h8go3HNgN0WU" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rH79t30YKs7i" + }, + "source": [ + "## 3 · Train through WeightsLab\n", + "\n", + "We register the run's hyperparameters with `wl.watch_or_edit(...)` (so they become live-editable from Studio), start the backend services with `wl.serve()`, then hand training to `WLAwareTrainer` via the standard `YOLO(...).train(trainer=...)` entry point.\n", + "\n", + "> **Connect Weights Studio** — run `weightslab start` locally (opens `http://localhost:5173`) *before or during* training to watch the run live. On **Colab**, serve over a public tunnel instead: use `wl.serve(serving_grpc=True, serving_bore=True)` and connect your local Studio with `weightslab tunnel bore.pub:PORT` (the port is printed when serving starts).\n", + "\n", + "Data augmentations are disabled so each sample maps cleanly to its ground truth in the grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ.setdefault(\"WL_PRELOAD_IMAGE_OVERVIEW\", \"0\")\n", + "os.environ.setdefault(\"WEIGHTSLAB_LOG_LEVEL\", \"WARNING\")\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "from weightslab.integrations.ultralytics import WLAwareSegmentationTrainer\n", + "from ultralytics import YOLO\n", + "\n", + "# Hyperparameters registered with WeightsLab -> live-editable from Weights Studio.\n", + "cfg = {\n", + " \"model\": \"yolo11n-seg.pt\", # pretrained checkpoint\n", + " \"data\": \"carparts-seg.yaml\", # Ultralytics auto-downloads\n", + " \"image_size\": 640,\n", + " \"epochs\": 10,\n", + " # Per-sample prediction overlays streamed to Studio (NMS on the train set).\n", + " \"signals_cfg\": {\n", + " \"train_nms\": {\"conf_thres\": 0.25, \"iou_thres\": 0.45, \"max_nms\": 7},\n", + " },\n", + "}\n", + "wl.watch_or_edit(cfg, flag=\"hyperparameters\", defaults=cfg, poll_interval=1.0)\n", + "\n", + "# Start the WeightsLab backend services that Weights Studio connects to.\n", + "wl.serve(serving_grpc=True)\n", + "# Colab -> local Studio: wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "wl.start_training() # equivalent to pressing \"play\" in Studio\n", + "\n", + "# Read back the (now live) hyperparameters and hand training to WLAwareSegmentationTrainer.\n", + "model = YOLO(cfg[\"model\"])\n", + "results = model.train(\n", + " trainer=WLAwareSegmentationTrainer,\n", + " data=str(cfg[\"data\"]),\n", + " imgsz=cfg[\"image_size\"],\n", + " epochs=int(cfg[\"epochs\"]),\n", + " project=\"./wl_logs\", name=\"carparts-seg\", # -> WL log_dir/name\n", + " workers=0, # WL invariant (parent-process uid counter)\n", + " optimizer=\"SGD\", lr0=0.001, amp=False,\n", + " # All augmentations off for a clean sample <-> ground-truth association.\n", + " mosaic=0.0, mixup=0.0, copy_paste=0.0,\n", + " hsv_h=0.0, hsv_s=0.0, hsv_v=0.0,\n", + " degrees=0.0, translate=0.0, scale=0.0, shear=0.0, perspective=0.0,\n", + " flipud=0.0, fliplr=0.0, erasing=0.0, auto_augment=None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f-Uw_s_8Ks7m" + }, + "source": [ + "## 4 · Explore the data grid\n", + "\n", + "The table below is exactly the data behind Weights Studio's **grid explorer**: one row per sample, keyed by split (`origin`) and sample id, with the per-sample loss, prediction stats and tags accumulated during training. This is the ledger that Studio reads from — here we render it inline with pandas.\n", + "\n", + "The image thumbnails and bounding-box overlays themselves are rendered by **Weights Studio** (which streams full previews over gRPC); the link below opens the interactive grid there." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "collapsed": true, + "id": "uOJdXGA3Ks7o", + "outputId": "26652fa3-6a15-43ce-9d75-034eaba21ccd" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "1241 samples tracked\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " prediction prediction_raw \\\n", + "sample_id annotation_id \n", + "0 0 None None \n", + "1 0 None None \n", + "2 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "3 0 None None \n", + "4 0 None None \n", + "5 0 None None \n", + "6 0 None None \n", + "7 0 None None \n", + "8 0 None None \n", + "9 0 None None \n", + "10 0 None None \n", + "11 0 None None \n", + "12 0 None None \n", + "13 0 None None \n", + "14 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "15 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "16 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "17 0 None None \n", + "18 0 None None \n", + "19 0 None None \n", + "20 0 None None \n", + " 1 None None \n", + " 2 None None \n", + " 3 None None \n", + "21 0 None None \n", + "22 0 None None \n", + "23 0 None None \n", + "24 0 None None \n", + "25 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "26 0 None None \n", + "27 0 None None \n", + "28 0 None None \n", + "29 0 None None \n", + "30 0 None None \n", + "31 0 None None \n", + "32 0 None None \n", + "33 0 None None \n", + "34 0 None None \n", + "35 0 None None \n", + "36 0 None None \n", + "\n", + " target \\\n", + "sample_id annotation_id \n", + "0 0 [0.2335685, 0.254695, 0.4553995, 0.43075103, 1... \n", + "1 0 [0.251174, 0.24061048, 0.44366202, 0.4307515, ... \n", + "2 0 None \n", + " 1 [0.523474, 0.3978875, 0.634976, 0.48122054, 1.... \n", + " 2 [0.40610352, 0.415493, 0.5234745, 0.522301, 1.... \n", + "3 0 [0.403756, 0.3826295, 0.637324, 0.5152585, 1.0... \n", + "4 0 [0.4436615, 0.3814555, 0.5938965, 0.4518785, 1... \n", + "5 0 [0.6044605, 0.3990615, 0.67253554, 0.46596247,... \n", + "6 0 [0.410798, 0.4436625, 0.556338, 0.51173747, 1.... \n", + "7 0 [0.650235, 0.4166665, 0.73826295, 0.48356748, ... \n", + "8 0 [0.64788747, 0.388498, 0.7582165, 0.49413198, ... \n", + "9 0 [0.6807515, 0.40140802, 0.75704247, 0.45774597... \n", + "10 0 [0.431925, 0.1701875, 0.561033, 0.3180745, 1.0... \n", + "11 0 [0.42018852, 0.34507048, 0.5434275, 0.4941315,... \n", + "12 0 [0.3732395, 0.348592, 0.54812247, 0.46009403, ... \n", + "13 0 [0.37793398, 0.3814555, 0.48122, 0.45892054, 1... \n", + "14 0 None \n", + " 1 [0.43427247, 0.3556335, 0.51173747, 0.43075046... \n", + " 2 [0.4929575, 0.44718304, 0.5375585, 0.485915, 1... \n", + "15 0 None \n", + " 1 [0.629108, 0.39788702, 0.67840403, 0.453051, 1... \n", + " 2 [0.4530515, 0.4107985, 0.58568054, 0.5105635, ... \n", + "16 0 None \n", + " 1 [0.45070451, 0.3967135, 0.5762915, 0.5152585, ... \n", + " 2 [0.63732404, 0.403756, 0.684272, 0.46009403, 1... \n", + "17 0 [0.43779355, 0.3826295, 0.61032856, 0.5223005,... \n", + "18 0 [0.4401405, 0.374413, 0.6068075, 0.529343, 1.0... \n", + "19 0 [0.536385, 0.4495305, 0.600939, 0.49999946, 0.... \n", + "20 0 None \n", + " 1 [0.52347445, 0.4424885, 0.6150235, 0.5281695, ... \n", + " 2 [0.456573, 0.515258, 0.504695, 0.562206, 0.0, ... \n", + " 3 [0.504695, 0.44248852, 0.54342705, 0.48004752,... \n", + "21 0 [0.54107946, 0.456573, 0.6103285, 0.511737, 0.... \n", + "22 0 [0.3298125, 0.2042255, 0.4225355, 0.2957745, 0... \n", + "23 0 [0.517606, 0.18661949, 0.615024, 0.2875585, 1.... \n", + "24 0 [0.51173747, 0.18779299, 0.6126765, 0.308685, ... \n", + "25 0 None \n", + " 1 [0.30985948, 0.3626765, 0.3861505, 0.43896753,... \n", + " 2 [0.41314548, 0.3427225, 0.48122054, 0.42018753... \n", + "26 0 [0.25821602, 0.30046952, 0.46830997, 0.4577465... \n", + "27 0 [0.2734745, 0.2828635, 0.44718352, 0.4577465, ... \n", + "28 0 [0.5915495, 0.23591551, 0.6455405, 0.2887325, ... \n", + "29 0 [0.5903755, 0.231221, 0.6420185, 0.280517, 1.0... \n", + "30 0 [0.40493003, 0.2018775, 0.469484, 0.2699525, 1... \n", + "31 0 [0.39554, 0.19483551, 0.483568, 0.2887325, 1.0... \n", + "32 0 [0.4025825, 0.212441, 0.45539945, 0.269953, 1.... \n", + "33 0 [0.2265255, 0.2136145, 0.3438965, 0.30751148, ... \n", + "34 0 [0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0] \n", + "35 0 [0.6079815, 0.320423, 0.7969485, 0.489437, 1.0... \n", + "36 0 [0.6185445, 0.30164298, 0.7758215, 0.494131, 1... \n", + "\n", + " discarded origin task_type last_seen group_id \\\n", + "sample_id annotation_id \n", + "0 0 False train_loader -1.0 0 \n", + "1 0 False train_loader -1.0 1 \n", + "2 0 False train_loader -1.0 2 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "3 0 False train_loader -1.0 3 \n", + "4 0 False train_loader -1.0 4 \n", + "5 0 False train_loader -1.0 5 \n", + "6 0 False train_loader -1.0 6 \n", + "7 0 False train_loader -1.0 7 \n", + "8 0 False train_loader -1.0 8 \n", + "9 0 False train_loader -1.0 9 \n", + "10 0 False train_loader -1.0 10 \n", + "11 0 False train_loader -1.0 11 \n", + "12 0 False train_loader -1.0 12 \n", + "13 0 False train_loader -1.0 13 \n", + "14 0 False train_loader -1.0 14 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "15 0 False train_loader -1.0 15 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "16 0 False train_loader -1.0 16 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "17 0 False train_loader -1.0 17 \n", + "18 0 False train_loader -1.0 18 \n", + "19 0 False train_loader -1.0 19 \n", + "20 0 False train_loader -1.0 20 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + " 3 None NaN None NaN None \n", + "21 0 False train_loader -1.0 21 \n", + "22 0 False train_loader -1.0 22 \n", + "23 0 False train_loader -1.0 23 \n", + "24 0 False train_loader -1.0 24 \n", + "25 0 False train_loader -1.0 25 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "26 0 False train_loader -1.0 26 \n", + "27 0 False train_loader -1.0 27 \n", + "28 0 False train_loader -1.0 28 \n", + "29 0 False train_loader -1.0 29 \n", + "30 0 False train_loader -1.0 30 \n", + "31 0 False train_loader -1.0 31 \n", + "32 0 False train_loader -1.0 32 \n", + "33 0 False train_loader -1.0 33 \n", + "34 0 False train_loader -1.0 34 \n", + "35 0 False train_loader -1.0 35 \n", + "36 0 False train_loader -1.0 36 \n", + "\n", + " member_rank \\\n", + "sample_id annotation_id \n", + "0 0 0.0 \n", + "1 0 0.0 \n", + "2 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "3 0 0.0 \n", + "4 0 0.0 \n", + "5 0 0.0 \n", + "6 0 0.0 \n", + "7 0 0.0 \n", + "8 0 0.0 \n", + "9 0 0.0 \n", + "10 0 0.0 \n", + "11 0 0.0 \n", + "12 0 0.0 \n", + "13 0 0.0 \n", + "14 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "15 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "16 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "17 0 0.0 \n", + "18 0 0.0 \n", + "19 0 0.0 \n", + "20 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + " 3 NaN \n", + "21 0 0.0 \n", + "22 0 0.0 \n", + "23 0 0.0 \n", + "24 0 0.0 \n", + "25 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "26 0 0.0 \n", + "27 0 0.0 \n", + "28 0 0.0 \n", + "29 0 0.0 \n", + "30 0 0.0 \n", + "31 0 0.0 \n", + "32 0 0.0 \n", + "33 0 0.0 \n", + "34 0 0.0 \n", + "35 0 0.0 \n", + "36 0 0.0 \n", + "\n", + " img_path \\\n", + "sample_id annotation_id \n", + "0 0 /content/datasets/brain-tumor/images/train/000... \n", + "1 0 /content/datasets/brain-tumor/images/train/000... \n", + "2 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "3 0 /content/datasets/brain-tumor/images/train/000... \n", + "4 0 /content/datasets/brain-tumor/images/train/000... \n", + "5 0 /content/datasets/brain-tumor/images/train/000... \n", + "6 0 /content/datasets/brain-tumor/images/train/000... \n", + "7 0 /content/datasets/brain-tumor/images/train/000... \n", + "8 0 /content/datasets/brain-tumor/images/train/000... \n", + "9 0 /content/datasets/brain-tumor/images/train/000... \n", + "10 0 /content/datasets/brain-tumor/images/train/000... \n", + "11 0 /content/datasets/brain-tumor/images/train/000... \n", + "12 0 /content/datasets/brain-tumor/images/train/000... \n", + "13 0 /content/datasets/brain-tumor/images/train/000... \n", + "14 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "15 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "16 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "17 0 /content/datasets/brain-tumor/images/train/000... \n", + "18 0 /content/datasets/brain-tumor/images/train/000... \n", + "19 0 /content/datasets/brain-tumor/images/train/000... \n", + "20 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 /content/datasets/brain-tumor/images/train/000... \n", + "22 0 /content/datasets/brain-tumor/images/train/000... \n", + "23 0 /content/datasets/brain-tumor/images/train/000... \n", + "24 0 /content/datasets/brain-tumor/images/train/000... \n", + "25 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "26 0 /content/datasets/brain-tumor/images/train/000... \n", + "27 0 /content/datasets/brain-tumor/images/train/000... \n", + "28 0 /content/datasets/brain-tumor/images/train/000... \n", + "29 0 /content/datasets/brain-tumor/images/train/000... \n", + "30 0 /content/datasets/brain-tumor/images/train/000... \n", + "31 0 /content/datasets/brain-tumor/images/train/000... \n", + "32 0 /content/datasets/brain-tumor/images/train/000... \n", + "33 0 /content/datasets/brain-tumor/images/train/000... \n", + "34 0 /content/datasets/brain-tumor/images/train/000... \n", + "35 0 /content/datasets/brain-tumor/images/train/000... \n", + "36 0 /content/datasets/brain-tumor/images/train/000... \n", + "\n", + " cls \n", + "sample_id annotation_id \n", + "0 0 [[1.0]] \n", + "1 0 [[1.0]] \n", + "2 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "3 0 [[1.0]] \n", + "4 0 [[1.0]] \n", + "5 0 [[1.0]] \n", + "6 0 [[1.0]] \n", + "7 0 [[1.0]] \n", + "8 0 [[1.0]] \n", + "9 0 [[1.0]] \n", + "10 0 [[1.0]] \n", + "11 0 [[1.0]] \n", + "12 0 [[1.0]] \n", + "13 0 [[1.0]] \n", + "14 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "15 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "16 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "17 0 [[1.0]] \n", + "18 0 [[1.0]] \n", + "19 0 [[0.0]] \n", + "20 0 [[0.0], [0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 [[0.0]] \n", + "22 0 [[0.0]] \n", + "23 0 [[1.0]] \n", + "24 0 [[1.0]] \n", + "25 0 [[0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + "26 0 [[0.0]] \n", + "27 0 [[0.0]] \n", + "28 0 [[1.0]] \n", + "29 0 [[1.0]] \n", + "30 0 [[1.0]] \n", + "31 0 [[1.0]] \n", + "32 0 [[1.0]] \n", + "33 0 [[1.0]] \n", + "34 0 [[1.0]] \n", + "35 0 [[1.0]] \n", + "36 0 [[1.0]] " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
predictionprediction_rawtargetdiscardedorigintask_typelast_seengroup_idmember_rankimg_pathcls
sample_idannotation_id
00NoneNone[0.2335685, 0.254695, 0.4553995, 0.43075103, 1...Falsetrain_loader-1.000.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
10NoneNone[0.251174, 0.24061048, 0.44366202, 0.4307515, ...Falsetrain_loader-1.010.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
20NoneNoneNoneFalsetrain_loader-1.020.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.523474, 0.3978875, 0.634976, 0.48122054, 1....NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.40610352, 0.415493, 0.5234745, 0.522301, 1....NoneNaNNoneNaNNoneNaNNoneNone
30NoneNone[0.403756, 0.3826295, 0.637324, 0.5152585, 1.0...Falsetrain_loader-1.030.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
40NoneNone[0.4436615, 0.3814555, 0.5938965, 0.4518785, 1...Falsetrain_loader-1.040.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
50NoneNone[0.6044605, 0.3990615, 0.67253554, 0.46596247,...Falsetrain_loader-1.050.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
60NoneNone[0.410798, 0.4436625, 0.556338, 0.51173747, 1....Falsetrain_loader-1.060.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
70NoneNone[0.650235, 0.4166665, 0.73826295, 0.48356748, ...Falsetrain_loader-1.070.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
80NoneNone[0.64788747, 0.388498, 0.7582165, 0.49413198, ...Falsetrain_loader-1.080.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
90NoneNone[0.6807515, 0.40140802, 0.75704247, 0.45774597...Falsetrain_loader-1.090.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
100NoneNone[0.431925, 0.1701875, 0.561033, 0.3180745, 1.0...Falsetrain_loader-1.0100.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
110NoneNone[0.42018852, 0.34507048, 0.5434275, 0.4941315,...Falsetrain_loader-1.0110.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
120NoneNone[0.3732395, 0.348592, 0.54812247, 0.46009403, ...Falsetrain_loader-1.0120.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
130NoneNone[0.37793398, 0.3814555, 0.48122, 0.45892054, 1...Falsetrain_loader-1.0130.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
140NoneNoneNoneFalsetrain_loader-1.0140.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.43427247, 0.3556335, 0.51173747, 0.43075046...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4929575, 0.44718304, 0.5375585, 0.485915, 1...NoneNaNNoneNaNNoneNaNNoneNone
150NoneNoneNoneFalsetrain_loader-1.0150.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.629108, 0.39788702, 0.67840403, 0.453051, 1...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4530515, 0.4107985, 0.58568054, 0.5105635, ...NoneNaNNoneNaNNoneNaNNoneNone
160NoneNoneNoneFalsetrain_loader-1.0160.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.45070451, 0.3967135, 0.5762915, 0.5152585, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.63732404, 0.403756, 0.684272, 0.46009403, 1...NoneNaNNoneNaNNoneNaNNoneNone
170NoneNone[0.43779355, 0.3826295, 0.61032856, 0.5223005,...Falsetrain_loader-1.0170.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
180NoneNone[0.4401405, 0.374413, 0.6068075, 0.529343, 1.0...Falsetrain_loader-1.0180.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
190NoneNone[0.536385, 0.4495305, 0.600939, 0.49999946, 0....Falsetrain_loader-1.0190.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
200NoneNoneNoneFalsetrain_loader-1.0200.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0], [0.0]]
1NoneNone[0.52347445, 0.4424885, 0.6150235, 0.5281695, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.456573, 0.515258, 0.504695, 0.562206, 0.0, ...NoneNaNNoneNaNNoneNaNNoneNone
3NoneNone[0.504695, 0.44248852, 0.54342705, 0.48004752,...NoneNaNNoneNaNNoneNaNNoneNone
210NoneNone[0.54107946, 0.456573, 0.6103285, 0.511737, 0....Falsetrain_loader-1.0210.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
220NoneNone[0.3298125, 0.2042255, 0.4225355, 0.2957745, 0...Falsetrain_loader-1.0220.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
230NoneNone[0.517606, 0.18661949, 0.615024, 0.2875585, 1....Falsetrain_loader-1.0230.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
240NoneNone[0.51173747, 0.18779299, 0.6126765, 0.308685, ...Falsetrain_loader-1.0240.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
250NoneNoneNoneFalsetrain_loader-1.0250.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0]]
1NoneNone[0.30985948, 0.3626765, 0.3861505, 0.43896753,...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.41314548, 0.3427225, 0.48122054, 0.42018753...NoneNaNNoneNaNNoneNaNNoneNone
260NoneNone[0.25821602, 0.30046952, 0.46830997, 0.4577465...Falsetrain_loader-1.0260.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
270NoneNone[0.2734745, 0.2828635, 0.44718352, 0.4577465, ...Falsetrain_loader-1.0270.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
280NoneNone[0.5915495, 0.23591551, 0.6455405, 0.2887325, ...Falsetrain_loader-1.0280.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
290NoneNone[0.5903755, 0.231221, 0.6420185, 0.280517, 1.0...Falsetrain_loader-1.0290.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
300NoneNone[0.40493003, 0.2018775, 0.469484, 0.2699525, 1...Falsetrain_loader-1.0300.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
310NoneNone[0.39554, 0.19483551, 0.483568, 0.2887325, 1.0...Falsetrain_loader-1.0310.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
320NoneNone[0.4025825, 0.212441, 0.45539945, 0.269953, 1....Falsetrain_loader-1.0320.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
330NoneNone[0.2265255, 0.2136145, 0.3438965, 0.30751148, ...Falsetrain_loader-1.0330.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
340NoneNone[0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0]Falsetrain_loader-1.0340.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
350NoneNone[0.6079815, 0.320423, 0.7969485, 0.489437, 1.0...Falsetrain_loader-1.0350.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
360NoneNone[0.6185445, 0.30164298, 0.7758215, 0.494131, 1...Falsetrain_loader-1.0360.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "repr_error": "Out of range float values are not JSON compliant: nan" + } + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "🔗 Open the full grid in Weights Studio" + ] + }, + "metadata": {} + } + ], + "source": [ + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "from weightslab.backend.ledgers import get_dataframe\n", + "\n", + "# `get_df_view()` is the same per-sample DataFrame that powers Studio's grid.\n", + "dfm = get_dataframe()\n", + "grid = dfm.get_df_view() if hasattr(dfm, \"get_df_view\") else pd.DataFrame()\n", + "print(f\"{len(grid)} samples tracked\")\n", + "display(grid.head(50))\n", + "\n", + "# Open the live, interactive grid (with image + bbox previews) in Weights Studio.\n", + "# Studio must be running locally (`weightslab start`).\n", + "display(HTML(\n", + " '🔗 Open the full grid in Weights Studio'\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FXQ0gBOSKs7t" + }, + "source": [ + "> **Can clicking a grid row here jump to that sample in Studio?** Not out of the box today. The notebook and Studio are separate processes, and the link above opens Studio at its root. Per-sample deep-linking (e.g. `http://localhost:5173/?sample=` selecting and scrolling to that row) is a small frontend addition to Weights Studio — the sample `uid` is already the grid's index here, so the notebook side is ready for it once Studio accepts the query param." + ] + } + ] +} diff --git a/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-construction-ppe-detection-dataset.ipynb b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-construction-ppe-detection-dataset.ipynb new file mode 100644 index 00000000..db630cac --- /dev/null +++ b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-construction-ppe-detection-dataset.ipynb @@ -0,0 +1,1716 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Train an Ultralytics YOLO detector on the construction-ppe dataset and stream every sample, prediction and metric live into Weights Studio to inspect, edit and evolve your run.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Construction PPE Detection · Ultralytics YOLO + WeightsLab\n", + "\n", + "This notebook trains a YOLO detector on the [construction-ppe](https://docs.ultralytics.com/) dataset (detecting personal protective equipment (helmets, vests, gloves, …) on construction sites) **through the WeightsLab training pipeline**.\n", + "\n", + "Instead of Ultralytics' one-shot `train`/`predict` flow, we hand training to `WLAwareTrainer`. It wraps the train/val dataloaders, logs a **per-sample** signal for every image (loss, prediction stats, tags) and ships live prediction overlays to **Weights Studio** — where you inspect the data grid, remove/weight samples, edit the model and resume, all while training runs.\n", + "\n", + "Ultralytics auto-downloads the dataset the first time you train." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o7YFKEN1Ks7Y" + }, + "source": [ + "## 1 · Setup\n", + "\n", + "Install WeightsLab (which pulls in Ultralytics) and verify the environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "71be394c", + "outputId": "c9ef4767-84f5-4bea-e395-9e11766002f2", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "source": [ + "%pip install setuptools wheel" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (75.2.0)\n", + "Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (0.47.0)\n", + "Requirement already satisfied: packaging>=24.0 in /usr/local/lib/python3.12/dist-packages (from wheel) (26.2)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "%pip install git+https://github.com/GrayboxTech/weightslab.git" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "jV1oi_PQN0xK", + "outputId": "eda8eca1-f4b3-4ffd-eaa6-0f67ce857153" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting git+https://github.com/GrayboxTech/weightslab.git\n", + " Cloning https://github.com/GrayboxTech/weightslab.git (to revision landingcollab) to /tmp/pip-req-build-z4ba7cx9\n", + " Running command git clone --filter=blob:none --quiet https://github.com/GrayboxTech/weightslab.git /tmp/pip-req-build-z4ba7cx9\n", + " Running command git checkout -b landingcollab --track origin/landingcollab\n", + " Switched to a new branch 'landingcollab'\n", + " Branch 'landingcollab' set up to track remote branch 'landingcollab' from 'origin'.\n", + " Resolved https://github.com/GrayboxTech/weightslab.git to commit 3f1cf143bc93e67df43adc71ab831fa71477e0d2\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.10.2)\n", + "Requirement already satisfied: torch<=2.9,>=2.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.9.0)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.24.0)\n", + "Requirement already satisfied: torchmetrics>=1.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.21)\n", + "Requirement already satisfied: onnx<=1.20,>=1.15 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.20.0)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.4.9)\n", + "Requirement already satisfied: langchain-ollama<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.1.0)\n", + "Requirement already satisfied: langchain-openai<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.5)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev8) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.2)\n", + "Requirement already satisfied: ollama<1.0.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev8) (0.6.2)\n", + "Requirement already satisfied: openai<3.0.0,>=2.45.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2.45.0)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev8) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.5.0)\n", + "Requirement already satisfied: lightning-utilities>=0.15.3 in /usr/local/lib/python3.12/dist-packages (from torchmetrics>=1.9->weightslab==1.3.3.dev8) (0.15.3)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.0.0)\n", + "Building wheels for collected packages: weightslab\n", + " Building wheel for weightslab (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for weightslab: filename=weightslab-1.3.3.dev8-py3-none-any.whl size=2827715 sha256=5c4e7c14bbf95a48b9a8c86b5a8c744af67e1eb395197cbe79d3e443c21cf86e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-__zwgf7x/wheels/4d/c7/cd/817c2745790555e7b6c532f5b6055d47567f9416fdfc65879b\n", + "Successfully built weightslab\n", + "Installing collected packages: weightslab\n", + " Attempting uninstall: weightslab\n", + " Found existing installation: weightslab 1.3.3.dev7\n", + " Uninstalling weightslab-1.3.3.dev7:\n", + " Successfully uninstalled weightslab-1.3.3.dev7\n", + "Successfully installed weightslab-1.3.3.dev8\n" + ] + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "weightslab" + ] + }, + "id": "b7fce274042740d19087972fe9c01c9b" + } + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pJhvREbaKs7f", + "outputId": "651baa36-9ebb-49da-830b-e6c8a6e2b64f" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Ultralytics 8.4.96 🚀 Python-3.12.13 torch-2.9.0+cu128 CPU (Intel Xeon CPU @ 2.20GHz)\n", + "Setup complete ✅ (2 CPUs, 12.7 GB RAM, 30.5/107.7 GB disk)\n", + "16/07/2026-09:29:29.683 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmp6meuzqdz/weightslab_logs/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:29.685 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-09:29:29.686 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev8 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-09:29:29.690 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install weightslab\n", + "# %pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev8\"\n", + "%pip install ultralytics\n", + "\n", + "import ultralytics\n", + "ultralytics.checks()\n", + "\n", + "import weightslab as wl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 · Dataset\n", + "\n", + "The dataset is described by a small YAML file that Ultralytics resolves and downloads automatically. For reference:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h8go3HNgN0WU" + }, + "source": [ + "```yaml\n", + "# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license\n", + "\n", + "# Construction-PPE dataset by Ultralytics\n", + "# Documentation: https://docs.ultralytics.com/datasets/detect/construction-ppe/\n", + "# Example usage: yolo train data=construction-ppe.yaml\n", + "# parent\n", + "# ├── ultralytics\n", + "# └── datasets\n", + "# └── construction-ppe ← downloads here (178.4 MB)\n", + "\n", + "# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..]\n", + "path: construction-ppe # dataset root dir\n", + "train: images/train # train images (relative to 'path') 1132 images\n", + "val: images/val # val images (relative to 'path') 143 images\n", + "test: images/test # test images (relative to 'path') 141 images\n", + "\n", + "# Classes\n", + "names:\n", + " 0: helmet\n", + " 1: gloves\n", + " 2: vest\n", + " 3: boots\n", + " 4: goggles\n", + " 5: none\n", + " 6: Person\n", + " 7: no_helmet\n", + " 8: no_goggle\n", + " 9: no_gloves\n", + " 10: no_boots\n", + "\n", + "# Download script/URL (optional)\n", + "download: https://github.com/ultralytics/assets/releases/download/v0.0.0/construction-ppe.zip\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rH79t30YKs7i" + }, + "source": [ + "## 3 · Train through WeightsLab\n", + "\n", + "We register the run's hyperparameters with `wl.watch_or_edit(...)` (so they become live-editable from Studio), start the backend services with `wl.serve()`, then hand training to `WLAwareTrainer` via the standard `YOLO(...).train(trainer=...)` entry point.\n", + "\n", + "> **Connect Weights Studio** — run `weightslab start` locally (opens `http://localhost:5173`) *before or during* training to watch the run live. On **Colab**, serve over a public tunnel instead: use `wl.serve(serving_grpc=True, serving_bore=True)` and connect your local Studio with `weightslab tunnel bore.pub:PORT` (the port is printed when serving starts).\n", + "\n", + "Data augmentations are disabled so each sample maps cleanly to its ground truth in the grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ.setdefault(\"WL_PRELOAD_IMAGE_OVERVIEW\", \"0\")\n", + "os.environ.setdefault(\"WEIGHTSLAB_LOG_LEVEL\", \"WARNING\")\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "from weightslab.integrations.ultralytics import WLAwareTrainer\n", + "from ultralytics import YOLO\n", + "\n", + "# Hyperparameters registered with WeightsLab -> live-editable from Weights Studio.\n", + "cfg = {\n", + " \"model\": \"yolo11n.pt\", # pretrained checkpoint\n", + " \"data\": \"construction-ppe.yaml\", # Ultralytics auto-downloads\n", + " \"image_size\": 640,\n", + " \"epochs\": 5,\n", + " # Per-sample prediction overlays streamed to Studio (NMS on the train set).\n", + " \"signals_cfg\": {\n", + " \"train_nms\": {\"conf_thres\": 0.25, \"iou_thres\": 0.45, \"max_nms\": 7},\n", + " },\n", + "}\n", + "wl.watch_or_edit(cfg, flag=\"hyperparameters\", defaults=cfg, poll_interval=1.0)\n", + "\n", + "# Start the WeightsLab backend services that Weights Studio connects to.\n", + "wl.serve(serving_grpc=True)\n", + "# Colab -> local Studio: wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "wl.start_training() # equivalent to pressing \"play\" in Studio\n", + "\n", + "# Read back the (now live) hyperparameters and hand training to WLAwareTrainer.\n", + "model = YOLO(cfg[\"model\"])\n", + "results = model.train(\n", + " trainer=WLAwareTrainer,\n", + " data=str(cfg[\"data\"]),\n", + " imgsz=cfg[\"image_size\"],\n", + " epochs=int(cfg[\"epochs\"]),\n", + " project=\"./wl_logs\", name=\"construction-ppe\", # -> WL log_dir/name\n", + " workers=0, # WL invariant (parent-process uid counter)\n", + " optimizer=\"SGD\", lr0=0.001, amp=False,\n", + " # All augmentations off for a clean sample <-> ground-truth association.\n", + " mosaic=0.0, mixup=0.0, copy_paste=0.0,\n", + " hsv_h=0.0, hsv_s=0.0, hsv_v=0.0,\n", + " degrees=0.0, translate=0.0, scale=0.0, shear=0.0, perspective=0.0,\n", + " flipud=0.0, fliplr=0.0, erasing=0.0, auto_augment=None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f-Uw_s_8Ks7m" + }, + "source": [ + "## 4 · Explore the data grid\n", + "\n", + "The table below is exactly the data behind Weights Studio's **grid explorer**: one row per sample, keyed by split (`origin`) and sample id, with the per-sample loss, prediction stats and tags accumulated during training. This is the ledger that Studio reads from — here we render it inline with pandas.\n", + "\n", + "The image thumbnails and bounding-box overlays themselves are rendered by **Weights Studio** (which streams full previews over gRPC); the link below opens the interactive grid there." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "collapsed": true, + "id": "uOJdXGA3Ks7o", + "outputId": "26652fa3-6a15-43ce-9d75-034eaba21ccd" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "1241 samples tracked\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " prediction prediction_raw \\\n", + "sample_id annotation_id \n", + "0 0 None None \n", + "1 0 None None \n", + "2 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "3 0 None None \n", + "4 0 None None \n", + "5 0 None None \n", + "6 0 None None \n", + "7 0 None None \n", + "8 0 None None \n", + "9 0 None None \n", + "10 0 None None \n", + "11 0 None None \n", + "12 0 None None \n", + "13 0 None None \n", + "14 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "15 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "16 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "17 0 None None \n", + "18 0 None None \n", + "19 0 None None \n", + "20 0 None None \n", + " 1 None None \n", + " 2 None None \n", + " 3 None None \n", + "21 0 None None \n", + "22 0 None None \n", + "23 0 None None \n", + "24 0 None None \n", + "25 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "26 0 None None \n", + "27 0 None None \n", + "28 0 None None \n", + "29 0 None None \n", + "30 0 None None \n", + "31 0 None None \n", + "32 0 None None \n", + "33 0 None None \n", + "34 0 None None \n", + "35 0 None None \n", + "36 0 None None \n", + "\n", + " target \\\n", + "sample_id annotation_id \n", + "0 0 [0.2335685, 0.254695, 0.4553995, 0.43075103, 1... \n", + "1 0 [0.251174, 0.24061048, 0.44366202, 0.4307515, ... \n", + "2 0 None \n", + " 1 [0.523474, 0.3978875, 0.634976, 0.48122054, 1.... \n", + " 2 [0.40610352, 0.415493, 0.5234745, 0.522301, 1.... \n", + "3 0 [0.403756, 0.3826295, 0.637324, 0.5152585, 1.0... \n", + "4 0 [0.4436615, 0.3814555, 0.5938965, 0.4518785, 1... \n", + "5 0 [0.6044605, 0.3990615, 0.67253554, 0.46596247,... \n", + "6 0 [0.410798, 0.4436625, 0.556338, 0.51173747, 1.... \n", + "7 0 [0.650235, 0.4166665, 0.73826295, 0.48356748, ... \n", + "8 0 [0.64788747, 0.388498, 0.7582165, 0.49413198, ... \n", + "9 0 [0.6807515, 0.40140802, 0.75704247, 0.45774597... \n", + "10 0 [0.431925, 0.1701875, 0.561033, 0.3180745, 1.0... \n", + "11 0 [0.42018852, 0.34507048, 0.5434275, 0.4941315,... \n", + "12 0 [0.3732395, 0.348592, 0.54812247, 0.46009403, ... \n", + "13 0 [0.37793398, 0.3814555, 0.48122, 0.45892054, 1... \n", + "14 0 None \n", + " 1 [0.43427247, 0.3556335, 0.51173747, 0.43075046... \n", + " 2 [0.4929575, 0.44718304, 0.5375585, 0.485915, 1... \n", + "15 0 None \n", + " 1 [0.629108, 0.39788702, 0.67840403, 0.453051, 1... \n", + " 2 [0.4530515, 0.4107985, 0.58568054, 0.5105635, ... \n", + "16 0 None \n", + " 1 [0.45070451, 0.3967135, 0.5762915, 0.5152585, ... \n", + " 2 [0.63732404, 0.403756, 0.684272, 0.46009403, 1... \n", + "17 0 [0.43779355, 0.3826295, 0.61032856, 0.5223005,... \n", + "18 0 [0.4401405, 0.374413, 0.6068075, 0.529343, 1.0... \n", + "19 0 [0.536385, 0.4495305, 0.600939, 0.49999946, 0.... \n", + "20 0 None \n", + " 1 [0.52347445, 0.4424885, 0.6150235, 0.5281695, ... \n", + " 2 [0.456573, 0.515258, 0.504695, 0.562206, 0.0, ... \n", + " 3 [0.504695, 0.44248852, 0.54342705, 0.48004752,... \n", + "21 0 [0.54107946, 0.456573, 0.6103285, 0.511737, 0.... \n", + "22 0 [0.3298125, 0.2042255, 0.4225355, 0.2957745, 0... \n", + "23 0 [0.517606, 0.18661949, 0.615024, 0.2875585, 1.... \n", + "24 0 [0.51173747, 0.18779299, 0.6126765, 0.308685, ... \n", + "25 0 None \n", + " 1 [0.30985948, 0.3626765, 0.3861505, 0.43896753,... \n", + " 2 [0.41314548, 0.3427225, 0.48122054, 0.42018753... \n", + "26 0 [0.25821602, 0.30046952, 0.46830997, 0.4577465... \n", + "27 0 [0.2734745, 0.2828635, 0.44718352, 0.4577465, ... \n", + "28 0 [0.5915495, 0.23591551, 0.6455405, 0.2887325, ... \n", + "29 0 [0.5903755, 0.231221, 0.6420185, 0.280517, 1.0... \n", + "30 0 [0.40493003, 0.2018775, 0.469484, 0.2699525, 1... \n", + "31 0 [0.39554, 0.19483551, 0.483568, 0.2887325, 1.0... \n", + "32 0 [0.4025825, 0.212441, 0.45539945, 0.269953, 1.... \n", + "33 0 [0.2265255, 0.2136145, 0.3438965, 0.30751148, ... \n", + "34 0 [0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0] \n", + "35 0 [0.6079815, 0.320423, 0.7969485, 0.489437, 1.0... \n", + "36 0 [0.6185445, 0.30164298, 0.7758215, 0.494131, 1... \n", + "\n", + " discarded origin task_type last_seen group_id \\\n", + "sample_id annotation_id \n", + "0 0 False train_loader -1.0 0 \n", + "1 0 False train_loader -1.0 1 \n", + "2 0 False train_loader -1.0 2 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "3 0 False train_loader -1.0 3 \n", + "4 0 False train_loader -1.0 4 \n", + "5 0 False train_loader -1.0 5 \n", + "6 0 False train_loader -1.0 6 \n", + "7 0 False train_loader -1.0 7 \n", + "8 0 False train_loader -1.0 8 \n", + "9 0 False train_loader -1.0 9 \n", + "10 0 False train_loader -1.0 10 \n", + "11 0 False train_loader -1.0 11 \n", + "12 0 False train_loader -1.0 12 \n", + "13 0 False train_loader -1.0 13 \n", + "14 0 False train_loader -1.0 14 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "15 0 False train_loader -1.0 15 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "16 0 False train_loader -1.0 16 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "17 0 False train_loader -1.0 17 \n", + "18 0 False train_loader -1.0 18 \n", + "19 0 False train_loader -1.0 19 \n", + "20 0 False train_loader -1.0 20 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + " 3 None NaN None NaN None \n", + "21 0 False train_loader -1.0 21 \n", + "22 0 False train_loader -1.0 22 \n", + "23 0 False train_loader -1.0 23 \n", + "24 0 False train_loader -1.0 24 \n", + "25 0 False train_loader -1.0 25 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "26 0 False train_loader -1.0 26 \n", + "27 0 False train_loader -1.0 27 \n", + "28 0 False train_loader -1.0 28 \n", + "29 0 False train_loader -1.0 29 \n", + "30 0 False train_loader -1.0 30 \n", + "31 0 False train_loader -1.0 31 \n", + "32 0 False train_loader -1.0 32 \n", + "33 0 False train_loader -1.0 33 \n", + "34 0 False train_loader -1.0 34 \n", + "35 0 False train_loader -1.0 35 \n", + "36 0 False train_loader -1.0 36 \n", + "\n", + " member_rank \\\n", + "sample_id annotation_id \n", + "0 0 0.0 \n", + "1 0 0.0 \n", + "2 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "3 0 0.0 \n", + "4 0 0.0 \n", + "5 0 0.0 \n", + "6 0 0.0 \n", + "7 0 0.0 \n", + "8 0 0.0 \n", + "9 0 0.0 \n", + "10 0 0.0 \n", + "11 0 0.0 \n", + "12 0 0.0 \n", + "13 0 0.0 \n", + "14 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "15 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "16 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "17 0 0.0 \n", + "18 0 0.0 \n", + "19 0 0.0 \n", + "20 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + " 3 NaN \n", + "21 0 0.0 \n", + "22 0 0.0 \n", + "23 0 0.0 \n", + "24 0 0.0 \n", + "25 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "26 0 0.0 \n", + "27 0 0.0 \n", + "28 0 0.0 \n", + "29 0 0.0 \n", + "30 0 0.0 \n", + "31 0 0.0 \n", + "32 0 0.0 \n", + "33 0 0.0 \n", + "34 0 0.0 \n", + "35 0 0.0 \n", + "36 0 0.0 \n", + "\n", + " img_path \\\n", + "sample_id annotation_id \n", + "0 0 /content/datasets/brain-tumor/images/train/000... \n", + "1 0 /content/datasets/brain-tumor/images/train/000... \n", + "2 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "3 0 /content/datasets/brain-tumor/images/train/000... \n", + "4 0 /content/datasets/brain-tumor/images/train/000... \n", + "5 0 /content/datasets/brain-tumor/images/train/000... \n", + "6 0 /content/datasets/brain-tumor/images/train/000... \n", + "7 0 /content/datasets/brain-tumor/images/train/000... \n", + "8 0 /content/datasets/brain-tumor/images/train/000... \n", + "9 0 /content/datasets/brain-tumor/images/train/000... \n", + "10 0 /content/datasets/brain-tumor/images/train/000... \n", + "11 0 /content/datasets/brain-tumor/images/train/000... \n", + "12 0 /content/datasets/brain-tumor/images/train/000... \n", + "13 0 /content/datasets/brain-tumor/images/train/000... \n", + "14 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "15 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "16 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "17 0 /content/datasets/brain-tumor/images/train/000... \n", + "18 0 /content/datasets/brain-tumor/images/train/000... \n", + "19 0 /content/datasets/brain-tumor/images/train/000... \n", + "20 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 /content/datasets/brain-tumor/images/train/000... \n", + "22 0 /content/datasets/brain-tumor/images/train/000... \n", + "23 0 /content/datasets/brain-tumor/images/train/000... \n", + "24 0 /content/datasets/brain-tumor/images/train/000... \n", + "25 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "26 0 /content/datasets/brain-tumor/images/train/000... \n", + "27 0 /content/datasets/brain-tumor/images/train/000... \n", + "28 0 /content/datasets/brain-tumor/images/train/000... \n", + "29 0 /content/datasets/brain-tumor/images/train/000... \n", + "30 0 /content/datasets/brain-tumor/images/train/000... \n", + "31 0 /content/datasets/brain-tumor/images/train/000... \n", + "32 0 /content/datasets/brain-tumor/images/train/000... \n", + "33 0 /content/datasets/brain-tumor/images/train/000... \n", + "34 0 /content/datasets/brain-tumor/images/train/000... \n", + "35 0 /content/datasets/brain-tumor/images/train/000... \n", + "36 0 /content/datasets/brain-tumor/images/train/000... \n", + "\n", + " cls \n", + "sample_id annotation_id \n", + "0 0 [[1.0]] \n", + "1 0 [[1.0]] \n", + "2 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "3 0 [[1.0]] \n", + "4 0 [[1.0]] \n", + "5 0 [[1.0]] \n", + "6 0 [[1.0]] \n", + "7 0 [[1.0]] \n", + "8 0 [[1.0]] \n", + "9 0 [[1.0]] \n", + "10 0 [[1.0]] \n", + "11 0 [[1.0]] \n", + "12 0 [[1.0]] \n", + "13 0 [[1.0]] \n", + "14 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "15 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "16 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "17 0 [[1.0]] \n", + "18 0 [[1.0]] \n", + "19 0 [[0.0]] \n", + "20 0 [[0.0], [0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 [[0.0]] \n", + "22 0 [[0.0]] \n", + "23 0 [[1.0]] \n", + "24 0 [[1.0]] \n", + "25 0 [[0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + "26 0 [[0.0]] \n", + "27 0 [[0.0]] \n", + "28 0 [[1.0]] \n", + "29 0 [[1.0]] \n", + "30 0 [[1.0]] \n", + "31 0 [[1.0]] \n", + "32 0 [[1.0]] \n", + "33 0 [[1.0]] \n", + "34 0 [[1.0]] \n", + "35 0 [[1.0]] \n", + "36 0 [[1.0]] " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
predictionprediction_rawtargetdiscardedorigintask_typelast_seengroup_idmember_rankimg_pathcls
sample_idannotation_id
00NoneNone[0.2335685, 0.254695, 0.4553995, 0.43075103, 1...Falsetrain_loader-1.000.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
10NoneNone[0.251174, 0.24061048, 0.44366202, 0.4307515, ...Falsetrain_loader-1.010.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
20NoneNoneNoneFalsetrain_loader-1.020.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.523474, 0.3978875, 0.634976, 0.48122054, 1....NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.40610352, 0.415493, 0.5234745, 0.522301, 1....NoneNaNNoneNaNNoneNaNNoneNone
30NoneNone[0.403756, 0.3826295, 0.637324, 0.5152585, 1.0...Falsetrain_loader-1.030.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
40NoneNone[0.4436615, 0.3814555, 0.5938965, 0.4518785, 1...Falsetrain_loader-1.040.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
50NoneNone[0.6044605, 0.3990615, 0.67253554, 0.46596247,...Falsetrain_loader-1.050.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
60NoneNone[0.410798, 0.4436625, 0.556338, 0.51173747, 1....Falsetrain_loader-1.060.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
70NoneNone[0.650235, 0.4166665, 0.73826295, 0.48356748, ...Falsetrain_loader-1.070.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
80NoneNone[0.64788747, 0.388498, 0.7582165, 0.49413198, ...Falsetrain_loader-1.080.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
90NoneNone[0.6807515, 0.40140802, 0.75704247, 0.45774597...Falsetrain_loader-1.090.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
100NoneNone[0.431925, 0.1701875, 0.561033, 0.3180745, 1.0...Falsetrain_loader-1.0100.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
110NoneNone[0.42018852, 0.34507048, 0.5434275, 0.4941315,...Falsetrain_loader-1.0110.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
120NoneNone[0.3732395, 0.348592, 0.54812247, 0.46009403, ...Falsetrain_loader-1.0120.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
130NoneNone[0.37793398, 0.3814555, 0.48122, 0.45892054, 1...Falsetrain_loader-1.0130.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
140NoneNoneNoneFalsetrain_loader-1.0140.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.43427247, 0.3556335, 0.51173747, 0.43075046...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4929575, 0.44718304, 0.5375585, 0.485915, 1...NoneNaNNoneNaNNoneNaNNoneNone
150NoneNoneNoneFalsetrain_loader-1.0150.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.629108, 0.39788702, 0.67840403, 0.453051, 1...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4530515, 0.4107985, 0.58568054, 0.5105635, ...NoneNaNNoneNaNNoneNaNNoneNone
160NoneNoneNoneFalsetrain_loader-1.0160.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.45070451, 0.3967135, 0.5762915, 0.5152585, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.63732404, 0.403756, 0.684272, 0.46009403, 1...NoneNaNNoneNaNNoneNaNNoneNone
170NoneNone[0.43779355, 0.3826295, 0.61032856, 0.5223005,...Falsetrain_loader-1.0170.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
180NoneNone[0.4401405, 0.374413, 0.6068075, 0.529343, 1.0...Falsetrain_loader-1.0180.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
190NoneNone[0.536385, 0.4495305, 0.600939, 0.49999946, 0....Falsetrain_loader-1.0190.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
200NoneNoneNoneFalsetrain_loader-1.0200.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0], [0.0]]
1NoneNone[0.52347445, 0.4424885, 0.6150235, 0.5281695, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.456573, 0.515258, 0.504695, 0.562206, 0.0, ...NoneNaNNoneNaNNoneNaNNoneNone
3NoneNone[0.504695, 0.44248852, 0.54342705, 0.48004752,...NoneNaNNoneNaNNoneNaNNoneNone
210NoneNone[0.54107946, 0.456573, 0.6103285, 0.511737, 0....Falsetrain_loader-1.0210.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
220NoneNone[0.3298125, 0.2042255, 0.4225355, 0.2957745, 0...Falsetrain_loader-1.0220.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
230NoneNone[0.517606, 0.18661949, 0.615024, 0.2875585, 1....Falsetrain_loader-1.0230.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
240NoneNone[0.51173747, 0.18779299, 0.6126765, 0.308685, ...Falsetrain_loader-1.0240.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
250NoneNoneNoneFalsetrain_loader-1.0250.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0]]
1NoneNone[0.30985948, 0.3626765, 0.3861505, 0.43896753,...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.41314548, 0.3427225, 0.48122054, 0.42018753...NoneNaNNoneNaNNoneNaNNoneNone
260NoneNone[0.25821602, 0.30046952, 0.46830997, 0.4577465...Falsetrain_loader-1.0260.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
270NoneNone[0.2734745, 0.2828635, 0.44718352, 0.4577465, ...Falsetrain_loader-1.0270.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
280NoneNone[0.5915495, 0.23591551, 0.6455405, 0.2887325, ...Falsetrain_loader-1.0280.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
290NoneNone[0.5903755, 0.231221, 0.6420185, 0.280517, 1.0...Falsetrain_loader-1.0290.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
300NoneNone[0.40493003, 0.2018775, 0.469484, 0.2699525, 1...Falsetrain_loader-1.0300.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
310NoneNone[0.39554, 0.19483551, 0.483568, 0.2887325, 1.0...Falsetrain_loader-1.0310.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
320NoneNone[0.4025825, 0.212441, 0.45539945, 0.269953, 1....Falsetrain_loader-1.0320.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
330NoneNone[0.2265255, 0.2136145, 0.3438965, 0.30751148, ...Falsetrain_loader-1.0330.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
340NoneNone[0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0]Falsetrain_loader-1.0340.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
350NoneNone[0.6079815, 0.320423, 0.7969485, 0.489437, 1.0...Falsetrain_loader-1.0350.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
360NoneNone[0.6185445, 0.30164298, 0.7758215, 0.494131, 1...Falsetrain_loader-1.0360.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "repr_error": "Out of range float values are not JSON compliant: nan" + } + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "🔗 Open the full grid in Weights Studio" + ] + }, + "metadata": {} + } + ], + "source": [ + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "from weightslab.backend.ledgers import get_dataframe\n", + "\n", + "# `get_df_view()` is the same per-sample DataFrame that powers Studio's grid.\n", + "dfm = get_dataframe()\n", + "grid = dfm.get_df_view() if hasattr(dfm, \"get_df_view\") else pd.DataFrame()\n", + "print(f\"{len(grid)} samples tracked\")\n", + "display(grid.head(50))\n", + "\n", + "# Open the live, interactive grid (with image + bbox previews) in Weights Studio.\n", + "# Studio must be running locally (`weightslab start`).\n", + "display(HTML(\n", + " '🔗 Open the full grid in Weights Studio'\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FXQ0gBOSKs7t" + }, + "source": [ + "> **Can clicking a grid row here jump to that sample in Studio?** Not out of the box today. The notebook and Studio are separate processes, and the link above opens Studio at its root. Per-sample deep-linking (e.g. `http://localhost:5173/?sample=` selecting and scrolling to that row) is a small frontend addition to Weights Studio — the sample `uid` is already the grid's index here, so the notebook side is ready for it once Studio accepts the query param." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vlHp09Nueb3d" + }, + "source": [ + "## Citation,License and Attribution\n", + "\n", + "```bibtex\n", + "@dataset{Dalvi_Construction_PPE_Dataset_2025,\n", + " author = {Mrunmayee Dalvi and Niyati Singh and Sahil Bhingarde and Ketaki Chalke},\n", + " title = {Construction-PPE: Personal Protective Equipment Detection Dataset},\n", + " month = {January},\n", + " year = {2025},\n", + " version = {1.0.0},\n", + " license = {AGPL-3.0},\n", + " url = {https://docs.ultralytics.com/datasets/detect/construction-ppe/},\n", + " publisher = {Ultralytics}\n", + "}\n", + "```\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-crack-segmentation-dataset.ipynb b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-crack-segmentation-dataset.ipynb new file mode 100644 index 00000000..1b189b0c --- /dev/null +++ b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-crack-segmentation-dataset.ipynb @@ -0,0 +1,1684 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Train an Ultralytics YOLO segmentation model on the crack-seg dataset and stream every sample, prediction and metric live into Weights Studio to inspect, edit and evolve your run.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Crack Segmentation · Ultralytics YOLO + WeightsLab\n", + "\n", + "This notebook trains a YOLO segmentation model on the [crack-seg](https://docs.ultralytics.com/) dataset (segmenting cracks in concrete/asphalt for infrastructure inspection) **through the WeightsLab training pipeline**.\n", + "\n", + "Instead of Ultralytics' one-shot `train`/`predict` flow, we hand training to `WLAwareSegmentationTrainer`. It wraps the train/val dataloaders, logs a **per-sample** signal for every image (loss, prediction stats, tags) and ships live prediction overlays to **Weights Studio** — where you inspect the data grid, remove/weight samples, edit the model and resume, all while training runs.\n", + "\n", + "Ultralytics auto-downloads the dataset the first time you train." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o7YFKEN1Ks7Y" + }, + "source": [ + "## 1 · Setup\n", + "\n", + "Install WeightsLab (which pulls in Ultralytics) and verify the environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "71be394c", + "outputId": "c9ef4767-84f5-4bea-e395-9e11766002f2", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "source": [ + "%pip install setuptools wheel" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (75.2.0)\n", + "Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (0.47.0)\n", + "Requirement already satisfied: packaging>=24.0 in /usr/local/lib/python3.12/dist-packages (from wheel) (26.2)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "%pip install git+https://github.com/GrayboxTech/weightslab.git" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "jV1oi_PQN0xK", + "outputId": "eda8eca1-f4b3-4ffd-eaa6-0f67ce857153" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting git+https://github.com/GrayboxTech/weightslab.git\n", + " Cloning https://github.com/GrayboxTech/weightslab.git (to revision landingcollab) to /tmp/pip-req-build-z4ba7cx9\n", + " Running command git clone --filter=blob:none --quiet https://github.com/GrayboxTech/weightslab.git /tmp/pip-req-build-z4ba7cx9\n", + " Running command git checkout -b landingcollab --track origin/landingcollab\n", + " Switched to a new branch 'landingcollab'\n", + " Branch 'landingcollab' set up to track remote branch 'landingcollab' from 'origin'.\n", + " Resolved https://github.com/GrayboxTech/weightslab.git to commit 3f1cf143bc93e67df43adc71ab831fa71477e0d2\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.10.2)\n", + "Requirement already satisfied: torch<=2.9,>=2.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.9.0)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.24.0)\n", + "Requirement already satisfied: torchmetrics>=1.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.21)\n", + "Requirement already satisfied: onnx<=1.20,>=1.15 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.20.0)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.4.9)\n", + "Requirement already satisfied: langchain-ollama<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.1.0)\n", + "Requirement already satisfied: langchain-openai<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.5)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev8) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.2)\n", + "Requirement already satisfied: ollama<1.0.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev8) (0.6.2)\n", + "Requirement already satisfied: openai<3.0.0,>=2.45.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2.45.0)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev8) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.5.0)\n", + "Requirement already satisfied: lightning-utilities>=0.15.3 in /usr/local/lib/python3.12/dist-packages (from torchmetrics>=1.9->weightslab==1.3.3.dev8) (0.15.3)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.0.0)\n", + "Building wheels for collected packages: weightslab\n", + " Building wheel for weightslab (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for weightslab: filename=weightslab-1.3.3.dev8-py3-none-any.whl size=2827715 sha256=5c4e7c14bbf95a48b9a8c86b5a8c744af67e1eb395197cbe79d3e443c21cf86e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-__zwgf7x/wheels/4d/c7/cd/817c2745790555e7b6c532f5b6055d47567f9416fdfc65879b\n", + "Successfully built weightslab\n", + "Installing collected packages: weightslab\n", + " Attempting uninstall: weightslab\n", + " Found existing installation: weightslab 1.3.3.dev7\n", + " Uninstalling weightslab-1.3.3.dev7:\n", + " Successfully uninstalled weightslab-1.3.3.dev7\n", + "Successfully installed weightslab-1.3.3.dev8\n" + ] + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "weightslab" + ] + }, + "id": "b7fce274042740d19087972fe9c01c9b" + } + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pJhvREbaKs7f", + "outputId": "651baa36-9ebb-49da-830b-e6c8a6e2b64f" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Ultralytics 8.4.96 🚀 Python-3.12.13 torch-2.9.0+cu128 CPU (Intel Xeon CPU @ 2.20GHz)\n", + "Setup complete ✅ (2 CPUs, 12.7 GB RAM, 30.5/107.7 GB disk)\n", + "16/07/2026-09:29:29.683 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmp6meuzqdz/weightslab_logs/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:29.685 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-09:29:29.686 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev8 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-09:29:29.690 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install weightslab\n", + "# %pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev8\"\n", + "%pip install ultralytics\n", + "\n", + "import ultralytics\n", + "ultralytics.checks()\n", + "\n", + "import weightslab as wl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 · Dataset\n", + "\n", + "The dataset is described by a small YAML file that Ultralytics resolves and downloads automatically. For reference:" + ] + }, + { + "cell_type": "markdown", + "source": [ + "```yaml\n", + "# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license\n", + "\n", + "# Crack-seg dataset by Ultralytics\n", + "# Documentation: https://docs.ultralytics.com/datasets/segment/crack-seg/\n", + "# Example usage: yolo train data=crack-seg.yaml\n", + "# parent\n", + "# ├── ultralytics\n", + "# └── datasets\n", + "# └── crack-seg ← downloads here (91.6 MB)\n", + "\n", + "# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..]\n", + "path: crack-seg # dataset root dir\n", + "train: images/train # train images (relative to 'path') 3717 images\n", + "val: images/val # val images (relative to 'path') 112 images\n", + "test: images/test # test images (relative to 'path') 200 images\n", + "\n", + "# Classes\n", + "names:\n", + " 0: crack\n", + "\n", + "# Download script/URL (optional)\n", + "download: https://github.com/ultralytics/assets/releases/download/v0.0.0/crack-seg.zip\n", + "```" + ], + "metadata": { + "id": "h8go3HNgN0WU" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rH79t30YKs7i" + }, + "source": [ + "## 3 · Train through WeightsLab\n", + "\n", + "We register the run's hyperparameters with `wl.watch_or_edit(...)` (so they become live-editable from Studio), start the backend services with `wl.serve()`, then hand training to `WLAwareTrainer` via the standard `YOLO(...).train(trainer=...)` entry point.\n", + "\n", + "> **Connect Weights Studio** — run `weightslab start` locally (opens `http://localhost:5173`) *before or during* training to watch the run live. On **Colab**, serve over a public tunnel instead: use `wl.serve(serving_grpc=True, serving_bore=True)` and connect your local Studio with `weightslab tunnel bore.pub:PORT` (the port is printed when serving starts).\n", + "\n", + "Data augmentations are disabled so each sample maps cleanly to its ground truth in the grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ.setdefault(\"WL_PRELOAD_IMAGE_OVERVIEW\", \"0\")\n", + "os.environ.setdefault(\"WEIGHTSLAB_LOG_LEVEL\", \"WARNING\")\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "from weightslab.integrations.ultralytics import WLAwareSegmentationTrainer\n", + "from ultralytics import YOLO\n", + "\n", + "# Hyperparameters registered with WeightsLab -> live-editable from Weights Studio.\n", + "cfg = {\n", + " \"model\": \"yolo11n-seg.pt\", # pretrained checkpoint\n", + " \"data\": \"crack-seg.yaml\", # Ultralytics auto-downloads\n", + " \"image_size\": 640,\n", + " \"epochs\": 3,\n", + " # Per-sample prediction overlays streamed to Studio (NMS on the train set).\n", + " \"signals_cfg\": {\n", + " \"train_nms\": {\"conf_thres\": 0.25, \"iou_thres\": 0.45, \"max_nms\": 7},\n", + " },\n", + "}\n", + "wl.watch_or_edit(cfg, flag=\"hyperparameters\", defaults=cfg, poll_interval=1.0)\n", + "\n", + "# Start the WeightsLab backend services that Weights Studio connects to.\n", + "wl.serve(serving_grpc=True)\n", + "# Colab -> local Studio: wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "wl.start_training() # equivalent to pressing \"play\" in Studio\n", + "\n", + "# Read back the (now live) hyperparameters and hand training to WLAwareSegmentationTrainer.\n", + "model = YOLO(cfg[\"model\"])\n", + "results = model.train(\n", + " trainer=WLAwareSegmentationTrainer,\n", + " data=str(cfg[\"data\"]),\n", + " imgsz=cfg[\"image_size\"],\n", + " epochs=int(cfg[\"epochs\"]),\n", + " project=\"./wl_logs\", name=\"crack-seg\", # -> WL log_dir/name\n", + " workers=0, # WL invariant (parent-process uid counter)\n", + " optimizer=\"SGD\", lr0=0.001, amp=False,\n", + " # All augmentations off for a clean sample <-> ground-truth association.\n", + " mosaic=0.0, mixup=0.0, copy_paste=0.0,\n", + " hsv_h=0.0, hsv_s=0.0, hsv_v=0.0,\n", + " degrees=0.0, translate=0.0, scale=0.0, shear=0.0, perspective=0.0,\n", + " flipud=0.0, fliplr=0.0, erasing=0.0, auto_augment=None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f-Uw_s_8Ks7m" + }, + "source": [ + "## 4 · Explore the data grid\n", + "\n", + "The table below is exactly the data behind Weights Studio's **grid explorer**: one row per sample, keyed by split (`origin`) and sample id, with the per-sample loss, prediction stats and tags accumulated during training. This is the ledger that Studio reads from — here we render it inline with pandas.\n", + "\n", + "The image thumbnails and bounding-box overlays themselves are rendered by **Weights Studio** (which streams full previews over gRPC); the link below opens the interactive grid there." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "collapsed": true, + "id": "uOJdXGA3Ks7o", + "outputId": "26652fa3-6a15-43ce-9d75-034eaba21ccd" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "1241 samples tracked\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " prediction prediction_raw \\\n", + "sample_id annotation_id \n", + "0 0 None None \n", + "1 0 None None \n", + "2 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "3 0 None None \n", + "4 0 None None \n", + "5 0 None None \n", + "6 0 None None \n", + "7 0 None None \n", + "8 0 None None \n", + "9 0 None None \n", + "10 0 None None \n", + "11 0 None None \n", + "12 0 None None \n", + "13 0 None None \n", + "14 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "15 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "16 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "17 0 None None \n", + "18 0 None None \n", + "19 0 None None \n", + "20 0 None None \n", + " 1 None None \n", + " 2 None None \n", + " 3 None None \n", + "21 0 None None \n", + "22 0 None None \n", + "23 0 None None \n", + "24 0 None None \n", + "25 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "26 0 None None \n", + "27 0 None None \n", + "28 0 None None \n", + "29 0 None None \n", + "30 0 None None \n", + "31 0 None None \n", + "32 0 None None \n", + "33 0 None None \n", + "34 0 None None \n", + "35 0 None None \n", + "36 0 None None \n", + "\n", + " target \\\n", + "sample_id annotation_id \n", + "0 0 [0.2335685, 0.254695, 0.4553995, 0.43075103, 1... \n", + "1 0 [0.251174, 0.24061048, 0.44366202, 0.4307515, ... \n", + "2 0 None \n", + " 1 [0.523474, 0.3978875, 0.634976, 0.48122054, 1.... \n", + " 2 [0.40610352, 0.415493, 0.5234745, 0.522301, 1.... \n", + "3 0 [0.403756, 0.3826295, 0.637324, 0.5152585, 1.0... \n", + "4 0 [0.4436615, 0.3814555, 0.5938965, 0.4518785, 1... \n", + "5 0 [0.6044605, 0.3990615, 0.67253554, 0.46596247,... \n", + "6 0 [0.410798, 0.4436625, 0.556338, 0.51173747, 1.... \n", + "7 0 [0.650235, 0.4166665, 0.73826295, 0.48356748, ... \n", + "8 0 [0.64788747, 0.388498, 0.7582165, 0.49413198, ... \n", + "9 0 [0.6807515, 0.40140802, 0.75704247, 0.45774597... \n", + "10 0 [0.431925, 0.1701875, 0.561033, 0.3180745, 1.0... \n", + "11 0 [0.42018852, 0.34507048, 0.5434275, 0.4941315,... \n", + "12 0 [0.3732395, 0.348592, 0.54812247, 0.46009403, ... \n", + "13 0 [0.37793398, 0.3814555, 0.48122, 0.45892054, 1... \n", + "14 0 None \n", + " 1 [0.43427247, 0.3556335, 0.51173747, 0.43075046... \n", + " 2 [0.4929575, 0.44718304, 0.5375585, 0.485915, 1... \n", + "15 0 None \n", + " 1 [0.629108, 0.39788702, 0.67840403, 0.453051, 1... \n", + " 2 [0.4530515, 0.4107985, 0.58568054, 0.5105635, ... \n", + "16 0 None \n", + " 1 [0.45070451, 0.3967135, 0.5762915, 0.5152585, ... \n", + " 2 [0.63732404, 0.403756, 0.684272, 0.46009403, 1... \n", + "17 0 [0.43779355, 0.3826295, 0.61032856, 0.5223005,... \n", + "18 0 [0.4401405, 0.374413, 0.6068075, 0.529343, 1.0... \n", + "19 0 [0.536385, 0.4495305, 0.600939, 0.49999946, 0.... \n", + "20 0 None \n", + " 1 [0.52347445, 0.4424885, 0.6150235, 0.5281695, ... \n", + " 2 [0.456573, 0.515258, 0.504695, 0.562206, 0.0, ... \n", + " 3 [0.504695, 0.44248852, 0.54342705, 0.48004752,... \n", + "21 0 [0.54107946, 0.456573, 0.6103285, 0.511737, 0.... \n", + "22 0 [0.3298125, 0.2042255, 0.4225355, 0.2957745, 0... \n", + "23 0 [0.517606, 0.18661949, 0.615024, 0.2875585, 1.... \n", + "24 0 [0.51173747, 0.18779299, 0.6126765, 0.308685, ... \n", + "25 0 None \n", + " 1 [0.30985948, 0.3626765, 0.3861505, 0.43896753,... \n", + " 2 [0.41314548, 0.3427225, 0.48122054, 0.42018753... \n", + "26 0 [0.25821602, 0.30046952, 0.46830997, 0.4577465... \n", + "27 0 [0.2734745, 0.2828635, 0.44718352, 0.4577465, ... \n", + "28 0 [0.5915495, 0.23591551, 0.6455405, 0.2887325, ... \n", + "29 0 [0.5903755, 0.231221, 0.6420185, 0.280517, 1.0... \n", + "30 0 [0.40493003, 0.2018775, 0.469484, 0.2699525, 1... \n", + "31 0 [0.39554, 0.19483551, 0.483568, 0.2887325, 1.0... \n", + "32 0 [0.4025825, 0.212441, 0.45539945, 0.269953, 1.... \n", + "33 0 [0.2265255, 0.2136145, 0.3438965, 0.30751148, ... \n", + "34 0 [0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0] \n", + "35 0 [0.6079815, 0.320423, 0.7969485, 0.489437, 1.0... \n", + "36 0 [0.6185445, 0.30164298, 0.7758215, 0.494131, 1... \n", + "\n", + " discarded origin task_type last_seen group_id \\\n", + "sample_id annotation_id \n", + "0 0 False train_loader -1.0 0 \n", + "1 0 False train_loader -1.0 1 \n", + "2 0 False train_loader -1.0 2 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "3 0 False train_loader -1.0 3 \n", + "4 0 False train_loader -1.0 4 \n", + "5 0 False train_loader -1.0 5 \n", + "6 0 False train_loader -1.0 6 \n", + "7 0 False train_loader -1.0 7 \n", + "8 0 False train_loader -1.0 8 \n", + "9 0 False train_loader -1.0 9 \n", + "10 0 False train_loader -1.0 10 \n", + "11 0 False train_loader -1.0 11 \n", + "12 0 False train_loader -1.0 12 \n", + "13 0 False train_loader -1.0 13 \n", + "14 0 False train_loader -1.0 14 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "15 0 False train_loader -1.0 15 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "16 0 False train_loader -1.0 16 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "17 0 False train_loader -1.0 17 \n", + "18 0 False train_loader -1.0 18 \n", + "19 0 False train_loader -1.0 19 \n", + "20 0 False train_loader -1.0 20 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + " 3 None NaN None NaN None \n", + "21 0 False train_loader -1.0 21 \n", + "22 0 False train_loader -1.0 22 \n", + "23 0 False train_loader -1.0 23 \n", + "24 0 False train_loader -1.0 24 \n", + "25 0 False train_loader -1.0 25 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "26 0 False train_loader -1.0 26 \n", + "27 0 False train_loader -1.0 27 \n", + "28 0 False train_loader -1.0 28 \n", + "29 0 False train_loader -1.0 29 \n", + "30 0 False train_loader -1.0 30 \n", + "31 0 False train_loader -1.0 31 \n", + "32 0 False train_loader -1.0 32 \n", + "33 0 False train_loader -1.0 33 \n", + "34 0 False train_loader -1.0 34 \n", + "35 0 False train_loader -1.0 35 \n", + "36 0 False train_loader -1.0 36 \n", + "\n", + " member_rank \\\n", + "sample_id annotation_id \n", + "0 0 0.0 \n", + "1 0 0.0 \n", + "2 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "3 0 0.0 \n", + "4 0 0.0 \n", + "5 0 0.0 \n", + "6 0 0.0 \n", + "7 0 0.0 \n", + "8 0 0.0 \n", + "9 0 0.0 \n", + "10 0 0.0 \n", + "11 0 0.0 \n", + "12 0 0.0 \n", + "13 0 0.0 \n", + "14 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "15 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "16 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "17 0 0.0 \n", + "18 0 0.0 \n", + "19 0 0.0 \n", + "20 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + " 3 NaN \n", + "21 0 0.0 \n", + "22 0 0.0 \n", + "23 0 0.0 \n", + "24 0 0.0 \n", + "25 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "26 0 0.0 \n", + "27 0 0.0 \n", + "28 0 0.0 \n", + "29 0 0.0 \n", + "30 0 0.0 \n", + "31 0 0.0 \n", + "32 0 0.0 \n", + "33 0 0.0 \n", + "34 0 0.0 \n", + "35 0 0.0 \n", + "36 0 0.0 \n", + "\n", + " img_path \\\n", + "sample_id annotation_id \n", + "0 0 /content/datasets/brain-tumor/images/train/000... \n", + "1 0 /content/datasets/brain-tumor/images/train/000... \n", + "2 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "3 0 /content/datasets/brain-tumor/images/train/000... \n", + "4 0 /content/datasets/brain-tumor/images/train/000... \n", + "5 0 /content/datasets/brain-tumor/images/train/000... \n", + "6 0 /content/datasets/brain-tumor/images/train/000... \n", + "7 0 /content/datasets/brain-tumor/images/train/000... \n", + "8 0 /content/datasets/brain-tumor/images/train/000... \n", + "9 0 /content/datasets/brain-tumor/images/train/000... \n", + "10 0 /content/datasets/brain-tumor/images/train/000... \n", + "11 0 /content/datasets/brain-tumor/images/train/000... \n", + "12 0 /content/datasets/brain-tumor/images/train/000... \n", + "13 0 /content/datasets/brain-tumor/images/train/000... \n", + "14 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "15 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "16 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "17 0 /content/datasets/brain-tumor/images/train/000... \n", + "18 0 /content/datasets/brain-tumor/images/train/000... \n", + "19 0 /content/datasets/brain-tumor/images/train/000... \n", + "20 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 /content/datasets/brain-tumor/images/train/000... \n", + "22 0 /content/datasets/brain-tumor/images/train/000... \n", + "23 0 /content/datasets/brain-tumor/images/train/000... \n", + "24 0 /content/datasets/brain-tumor/images/train/000... \n", + "25 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "26 0 /content/datasets/brain-tumor/images/train/000... \n", + "27 0 /content/datasets/brain-tumor/images/train/000... \n", + "28 0 /content/datasets/brain-tumor/images/train/000... \n", + "29 0 /content/datasets/brain-tumor/images/train/000... \n", + "30 0 /content/datasets/brain-tumor/images/train/000... \n", + "31 0 /content/datasets/brain-tumor/images/train/000... \n", + "32 0 /content/datasets/brain-tumor/images/train/000... \n", + "33 0 /content/datasets/brain-tumor/images/train/000... \n", + "34 0 /content/datasets/brain-tumor/images/train/000... \n", + "35 0 /content/datasets/brain-tumor/images/train/000... \n", + "36 0 /content/datasets/brain-tumor/images/train/000... \n", + "\n", + " cls \n", + "sample_id annotation_id \n", + "0 0 [[1.0]] \n", + "1 0 [[1.0]] \n", + "2 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "3 0 [[1.0]] \n", + "4 0 [[1.0]] \n", + "5 0 [[1.0]] \n", + "6 0 [[1.0]] \n", + "7 0 [[1.0]] \n", + "8 0 [[1.0]] \n", + "9 0 [[1.0]] \n", + "10 0 [[1.0]] \n", + "11 0 [[1.0]] \n", + "12 0 [[1.0]] \n", + "13 0 [[1.0]] \n", + "14 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "15 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "16 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "17 0 [[1.0]] \n", + "18 0 [[1.0]] \n", + "19 0 [[0.0]] \n", + "20 0 [[0.0], [0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 [[0.0]] \n", + "22 0 [[0.0]] \n", + "23 0 [[1.0]] \n", + "24 0 [[1.0]] \n", + "25 0 [[0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + "26 0 [[0.0]] \n", + "27 0 [[0.0]] \n", + "28 0 [[1.0]] \n", + "29 0 [[1.0]] \n", + "30 0 [[1.0]] \n", + "31 0 [[1.0]] \n", + "32 0 [[1.0]] \n", + "33 0 [[1.0]] \n", + "34 0 [[1.0]] \n", + "35 0 [[1.0]] \n", + "36 0 [[1.0]] " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
predictionprediction_rawtargetdiscardedorigintask_typelast_seengroup_idmember_rankimg_pathcls
sample_idannotation_id
00NoneNone[0.2335685, 0.254695, 0.4553995, 0.43075103, 1...Falsetrain_loader-1.000.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
10NoneNone[0.251174, 0.24061048, 0.44366202, 0.4307515, ...Falsetrain_loader-1.010.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
20NoneNoneNoneFalsetrain_loader-1.020.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.523474, 0.3978875, 0.634976, 0.48122054, 1....NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.40610352, 0.415493, 0.5234745, 0.522301, 1....NoneNaNNoneNaNNoneNaNNoneNone
30NoneNone[0.403756, 0.3826295, 0.637324, 0.5152585, 1.0...Falsetrain_loader-1.030.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
40NoneNone[0.4436615, 0.3814555, 0.5938965, 0.4518785, 1...Falsetrain_loader-1.040.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
50NoneNone[0.6044605, 0.3990615, 0.67253554, 0.46596247,...Falsetrain_loader-1.050.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
60NoneNone[0.410798, 0.4436625, 0.556338, 0.51173747, 1....Falsetrain_loader-1.060.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
70NoneNone[0.650235, 0.4166665, 0.73826295, 0.48356748, ...Falsetrain_loader-1.070.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
80NoneNone[0.64788747, 0.388498, 0.7582165, 0.49413198, ...Falsetrain_loader-1.080.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
90NoneNone[0.6807515, 0.40140802, 0.75704247, 0.45774597...Falsetrain_loader-1.090.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
100NoneNone[0.431925, 0.1701875, 0.561033, 0.3180745, 1.0...Falsetrain_loader-1.0100.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
110NoneNone[0.42018852, 0.34507048, 0.5434275, 0.4941315,...Falsetrain_loader-1.0110.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
120NoneNone[0.3732395, 0.348592, 0.54812247, 0.46009403, ...Falsetrain_loader-1.0120.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
130NoneNone[0.37793398, 0.3814555, 0.48122, 0.45892054, 1...Falsetrain_loader-1.0130.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
140NoneNoneNoneFalsetrain_loader-1.0140.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.43427247, 0.3556335, 0.51173747, 0.43075046...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4929575, 0.44718304, 0.5375585, 0.485915, 1...NoneNaNNoneNaNNoneNaNNoneNone
150NoneNoneNoneFalsetrain_loader-1.0150.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.629108, 0.39788702, 0.67840403, 0.453051, 1...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4530515, 0.4107985, 0.58568054, 0.5105635, ...NoneNaNNoneNaNNoneNaNNoneNone
160NoneNoneNoneFalsetrain_loader-1.0160.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.45070451, 0.3967135, 0.5762915, 0.5152585, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.63732404, 0.403756, 0.684272, 0.46009403, 1...NoneNaNNoneNaNNoneNaNNoneNone
170NoneNone[0.43779355, 0.3826295, 0.61032856, 0.5223005,...Falsetrain_loader-1.0170.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
180NoneNone[0.4401405, 0.374413, 0.6068075, 0.529343, 1.0...Falsetrain_loader-1.0180.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
190NoneNone[0.536385, 0.4495305, 0.600939, 0.49999946, 0....Falsetrain_loader-1.0190.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
200NoneNoneNoneFalsetrain_loader-1.0200.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0], [0.0]]
1NoneNone[0.52347445, 0.4424885, 0.6150235, 0.5281695, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.456573, 0.515258, 0.504695, 0.562206, 0.0, ...NoneNaNNoneNaNNoneNaNNoneNone
3NoneNone[0.504695, 0.44248852, 0.54342705, 0.48004752,...NoneNaNNoneNaNNoneNaNNoneNone
210NoneNone[0.54107946, 0.456573, 0.6103285, 0.511737, 0....Falsetrain_loader-1.0210.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
220NoneNone[0.3298125, 0.2042255, 0.4225355, 0.2957745, 0...Falsetrain_loader-1.0220.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
230NoneNone[0.517606, 0.18661949, 0.615024, 0.2875585, 1....Falsetrain_loader-1.0230.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
240NoneNone[0.51173747, 0.18779299, 0.6126765, 0.308685, ...Falsetrain_loader-1.0240.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
250NoneNoneNoneFalsetrain_loader-1.0250.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0]]
1NoneNone[0.30985948, 0.3626765, 0.3861505, 0.43896753,...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.41314548, 0.3427225, 0.48122054, 0.42018753...NoneNaNNoneNaNNoneNaNNoneNone
260NoneNone[0.25821602, 0.30046952, 0.46830997, 0.4577465...Falsetrain_loader-1.0260.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
270NoneNone[0.2734745, 0.2828635, 0.44718352, 0.4577465, ...Falsetrain_loader-1.0270.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
280NoneNone[0.5915495, 0.23591551, 0.6455405, 0.2887325, ...Falsetrain_loader-1.0280.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
290NoneNone[0.5903755, 0.231221, 0.6420185, 0.280517, 1.0...Falsetrain_loader-1.0290.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
300NoneNone[0.40493003, 0.2018775, 0.469484, 0.2699525, 1...Falsetrain_loader-1.0300.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
310NoneNone[0.39554, 0.19483551, 0.483568, 0.2887325, 1.0...Falsetrain_loader-1.0310.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
320NoneNone[0.4025825, 0.212441, 0.45539945, 0.269953, 1....Falsetrain_loader-1.0320.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
330NoneNone[0.2265255, 0.2136145, 0.3438965, 0.30751148, ...Falsetrain_loader-1.0330.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
340NoneNone[0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0]Falsetrain_loader-1.0340.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
350NoneNone[0.6079815, 0.320423, 0.7969485, 0.489437, 1.0...Falsetrain_loader-1.0350.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
360NoneNone[0.6185445, 0.30164298, 0.7758215, 0.494131, 1...Falsetrain_loader-1.0360.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "repr_error": "Out of range float values are not JSON compliant: nan" + } + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "🔗 Open the full grid in Weights Studio" + ] + }, + "metadata": {} + } + ], + "source": [ + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "from weightslab.backend.ledgers import get_dataframe\n", + "\n", + "# `get_df_view()` is the same per-sample DataFrame that powers Studio's grid.\n", + "dfm = get_dataframe()\n", + "grid = dfm.get_df_view() if hasattr(dfm, \"get_df_view\") else pd.DataFrame()\n", + "print(f\"{len(grid)} samples tracked\")\n", + "display(grid.head(50))\n", + "\n", + "# Open the live, interactive grid (with image + bbox previews) in Weights Studio.\n", + "# Studio must be running locally (`weightslab start`).\n", + "display(HTML(\n", + " '🔗 Open the full grid in Weights Studio'\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FXQ0gBOSKs7t" + }, + "source": [ + "> **Can clicking a grid row here jump to that sample in Studio?** Not out of the box today. The notebook and Studio are separate processes, and the link above opens Studio at its root. Per-sample deep-linking (e.g. `http://localhost:5173/?sample=` selecting and scrolling to that row) is a small frontend addition to Weights Studio — the sample `uid` is already the grid's index here, so the notebook side is ready for it once Studio accepts the query param." + ] + } + ] +} diff --git a/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-homeobjects-dataset.ipynb b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-homeobjects-dataset.ipynb new file mode 100644 index 00000000..17d3c8bd --- /dev/null +++ b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-homeobjects-dataset.ipynb @@ -0,0 +1,1716 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Train an Ultralytics YOLO detector on the HomeObjects-3K dataset and stream every sample, prediction and metric live into Weights Studio to inspect, edit and evolve your run.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# HomeObjects-3K Detection · Ultralytics YOLO + WeightsLab\n", + "\n", + "This notebook trains a YOLO detector on the [HomeObjects-3K](https://docs.ultralytics.com/) dataset (detecting common indoor household objects) **through the WeightsLab training pipeline**.\n", + "\n", + "Instead of Ultralytics' one-shot `train`/`predict` flow, we hand training to `WLAwareTrainer`. It wraps the train/val dataloaders, logs a **per-sample** signal for every image (loss, prediction stats, tags) and ships live prediction overlays to **Weights Studio** — where you inspect the data grid, remove/weight samples, edit the model and resume, all while training runs.\n", + "\n", + "Ultralytics auto-downloads the dataset the first time you train." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o7YFKEN1Ks7Y" + }, + "source": [ + "## 1 · Setup\n", + "\n", + "Install WeightsLab (which pulls in Ultralytics) and verify the environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "71be394c", + "outputId": "c9ef4767-84f5-4bea-e395-9e11766002f2", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "source": [ + "%pip install setuptools wheel" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (75.2.0)\n", + "Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (0.47.0)\n", + "Requirement already satisfied: packaging>=24.0 in /usr/local/lib/python3.12/dist-packages (from wheel) (26.2)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "%pip install git+https://github.com/GrayboxTech/weightslab.git" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "jV1oi_PQN0xK", + "outputId": "eda8eca1-f4b3-4ffd-eaa6-0f67ce857153" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting git+https://github.com/GrayboxTech/weightslab.git\n", + " Cloning https://github.com/GrayboxTech/weightslab.git (to revision landingcollab) to /tmp/pip-req-build-z4ba7cx9\n", + " Running command git clone --filter=blob:none --quiet https://github.com/GrayboxTech/weightslab.git /tmp/pip-req-build-z4ba7cx9\n", + " Running command git checkout -b landingcollab --track origin/landingcollab\n", + " Switched to a new branch 'landingcollab'\n", + " Branch 'landingcollab' set up to track remote branch 'landingcollab' from 'origin'.\n", + " Resolved https://github.com/GrayboxTech/weightslab.git to commit 3f1cf143bc93e67df43adc71ab831fa71477e0d2\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.10.2)\n", + "Requirement already satisfied: torch<=2.9,>=2.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.9.0)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.24.0)\n", + "Requirement already satisfied: torchmetrics>=1.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.21)\n", + "Requirement already satisfied: onnx<=1.20,>=1.15 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.20.0)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.4.9)\n", + "Requirement already satisfied: langchain-ollama<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.1.0)\n", + "Requirement already satisfied: langchain-openai<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.5)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev8) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.2)\n", + "Requirement already satisfied: ollama<1.0.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev8) (0.6.2)\n", + "Requirement already satisfied: openai<3.0.0,>=2.45.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2.45.0)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev8) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.5.0)\n", + "Requirement already satisfied: lightning-utilities>=0.15.3 in /usr/local/lib/python3.12/dist-packages (from torchmetrics>=1.9->weightslab==1.3.3.dev8) (0.15.3)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.0.0)\n", + "Building wheels for collected packages: weightslab\n", + " Building wheel for weightslab (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for weightslab: filename=weightslab-1.3.3.dev8-py3-none-any.whl size=2827715 sha256=5c4e7c14bbf95a48b9a8c86b5a8c744af67e1eb395197cbe79d3e443c21cf86e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-__zwgf7x/wheels/4d/c7/cd/817c2745790555e7b6c532f5b6055d47567f9416fdfc65879b\n", + "Successfully built weightslab\n", + "Installing collected packages: weightslab\n", + " Attempting uninstall: weightslab\n", + " Found existing installation: weightslab 1.3.3.dev7\n", + " Uninstalling weightslab-1.3.3.dev7:\n", + " Successfully uninstalled weightslab-1.3.3.dev7\n", + "Successfully installed weightslab-1.3.3.dev8\n" + ] + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "weightslab" + ] + }, + "id": "b7fce274042740d19087972fe9c01c9b" + } + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pJhvREbaKs7f", + "outputId": "651baa36-9ebb-49da-830b-e6c8a6e2b64f" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Ultralytics 8.4.96 🚀 Python-3.12.13 torch-2.9.0+cu128 CPU (Intel Xeon CPU @ 2.20GHz)\n", + "Setup complete ✅ (2 CPUs, 12.7 GB RAM, 30.5/107.7 GB disk)\n", + "16/07/2026-09:29:29.683 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmp6meuzqdz/weightslab_logs/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:29.685 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-09:29:29.686 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev8 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-09:29:29.690 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install weightslab\n", + "# %pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev8\"\n", + "%pip install ultralytics\n", + "\n", + "import ultralytics\n", + "ultralytics.checks()\n", + "\n", + "import weightslab as wl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 · Dataset\n", + "\n", + "The dataset is described by a small YAML file that Ultralytics resolves and downloads automatically. For reference:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h8go3HNgN0WU" + }, + "source": [ + "```yaml\n", + "# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license\n", + "\n", + "# HomeObjects-3K dataset by Ultralytics\n", + "# Documentation: https://docs.ultralytics.com/datasets/detect/homeobjects-3k/\n", + "# Example usage: yolo train data=HomeObjects-3K.yaml\n", + "# parent\n", + "# ├── ultralytics\n", + "# └── datasets\n", + "# └── homeobjects-3K ← downloads here (390 MB)\n", + "\n", + "# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..]\n", + "path: homeobjects-3K # dataset root dir\n", + "train: images/train # train images (relative to 'path') 2285 images\n", + "val: images/val # val images (relative to 'path') 404 images\n", + "\n", + "# Classes\n", + "names:\n", + " 0: bed\n", + " 1: sofa\n", + " 2: chair\n", + " 3: table\n", + " 4: lamp\n", + " 5: tv\n", + " 6: laptop\n", + " 7: wardrobe\n", + " 8: window\n", + " 9: door\n", + " 10: potted plant\n", + " 11: photo frame\n", + "\n", + "# Download script/URL (optional)\n", + "download: https://github.com/ultralytics/assets/releases/download/v0.0.0/homeobjects-3K.zip\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rH79t30YKs7i" + }, + "source": [ + "## 3 · Train through WeightsLab\n", + "\n", + "We register the run's hyperparameters with `wl.watch_or_edit(...)` (so they become live-editable from Studio), start the backend services with `wl.serve()`, then hand training to `WLAwareTrainer` via the standard `YOLO(...).train(trainer=...)` entry point.\n", + "\n", + "> **Connect Weights Studio** — run `weightslab start` locally (opens `http://localhost:5173`) *before or during* training to watch the run live. On **Colab**, serve over a public tunnel instead: use `wl.serve(serving_grpc=True, serving_bore=True)` and connect your local Studio with `weightslab tunnel bore.pub:PORT` (the port is printed when serving starts).\n", + "\n", + "Data augmentations are disabled so each sample maps cleanly to its ground truth in the grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ.setdefault(\"WL_PRELOAD_IMAGE_OVERVIEW\", \"0\")\n", + "os.environ.setdefault(\"WEIGHTSLAB_LOG_LEVEL\", \"WARNING\")\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "from weightslab.integrations.ultralytics import WLAwareTrainer\n", + "from ultralytics import YOLO\n", + "\n", + "# Hyperparameters registered with WeightsLab -> live-editable from Weights Studio.\n", + "cfg = {\n", + " \"model\": \"yolo11n.pt\", # pretrained checkpoint\n", + " \"data\": \"HomeObjects-3K.yaml\", # Ultralytics auto-downloads\n", + " \"image_size\": 640,\n", + " \"epochs\": 3,\n", + " # Per-sample prediction overlays streamed to Studio (NMS on the train set).\n", + " \"signals_cfg\": {\n", + " \"train_nms\": {\"conf_thres\": 0.25, \"iou_thres\": 0.45, \"max_nms\": 7},\n", + " },\n", + "}\n", + "wl.watch_or_edit(cfg, flag=\"hyperparameters\", defaults=cfg, poll_interval=1.0)\n", + "\n", + "# Start the WeightsLab backend services that Weights Studio connects to.\n", + "wl.serve(serving_grpc=True)\n", + "# Colab -> local Studio: wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "wl.start_training() # equivalent to pressing \"play\" in Studio\n", + "\n", + "# Read back the (now live) hyperparameters and hand training to WLAwareTrainer.\n", + "model = YOLO(cfg[\"model\"])\n", + "results = model.train(\n", + " trainer=WLAwareTrainer,\n", + " data=str(cfg[\"data\"]),\n", + " imgsz=cfg[\"image_size\"],\n", + " epochs=int(cfg[\"epochs\"]),\n", + " project=\"./wl_logs\", name=\"homeobjects-3k\", # -> WL log_dir/name\n", + " workers=0, # WL invariant (parent-process uid counter)\n", + " optimizer=\"SGD\", lr0=0.001, amp=False,\n", + " # All augmentations off for a clean sample <-> ground-truth association.\n", + " mosaic=0.0, mixup=0.0, copy_paste=0.0,\n", + " hsv_h=0.0, hsv_s=0.0, hsv_v=0.0,\n", + " degrees=0.0, translate=0.0, scale=0.0, shear=0.0, perspective=0.0,\n", + " flipud=0.0, fliplr=0.0, erasing=0.0, auto_augment=None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f-Uw_s_8Ks7m" + }, + "source": [ + "## 4 · Explore the data grid\n", + "\n", + "The table below is exactly the data behind Weights Studio's **grid explorer**: one row per sample, keyed by split (`origin`) and sample id, with the per-sample loss, prediction stats and tags accumulated during training. This is the ledger that Studio reads from — here we render it inline with pandas.\n", + "\n", + "The image thumbnails and bounding-box overlays themselves are rendered by **Weights Studio** (which streams full previews over gRPC); the link below opens the interactive grid there." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "collapsed": true, + "id": "uOJdXGA3Ks7o", + "outputId": "26652fa3-6a15-43ce-9d75-034eaba21ccd" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "1241 samples tracked\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " prediction prediction_raw \\\n", + "sample_id annotation_id \n", + "0 0 None None \n", + "1 0 None None \n", + "2 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "3 0 None None \n", + "4 0 None None \n", + "5 0 None None \n", + "6 0 None None \n", + "7 0 None None \n", + "8 0 None None \n", + "9 0 None None \n", + "10 0 None None \n", + "11 0 None None \n", + "12 0 None None \n", + "13 0 None None \n", + "14 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "15 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "16 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "17 0 None None \n", + "18 0 None None \n", + "19 0 None None \n", + "20 0 None None \n", + " 1 None None \n", + " 2 None None \n", + " 3 None None \n", + "21 0 None None \n", + "22 0 None None \n", + "23 0 None None \n", + "24 0 None None \n", + "25 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "26 0 None None \n", + "27 0 None None \n", + "28 0 None None \n", + "29 0 None None \n", + "30 0 None None \n", + "31 0 None None \n", + "32 0 None None \n", + "33 0 None None \n", + "34 0 None None \n", + "35 0 None None \n", + "36 0 None None \n", + "\n", + " target \\\n", + "sample_id annotation_id \n", + "0 0 [0.2335685, 0.254695, 0.4553995, 0.43075103, 1... \n", + "1 0 [0.251174, 0.24061048, 0.44366202, 0.4307515, ... \n", + "2 0 None \n", + " 1 [0.523474, 0.3978875, 0.634976, 0.48122054, 1.... \n", + " 2 [0.40610352, 0.415493, 0.5234745, 0.522301, 1.... \n", + "3 0 [0.403756, 0.3826295, 0.637324, 0.5152585, 1.0... \n", + "4 0 [0.4436615, 0.3814555, 0.5938965, 0.4518785, 1... \n", + "5 0 [0.6044605, 0.3990615, 0.67253554, 0.46596247,... \n", + "6 0 [0.410798, 0.4436625, 0.556338, 0.51173747, 1.... \n", + "7 0 [0.650235, 0.4166665, 0.73826295, 0.48356748, ... \n", + "8 0 [0.64788747, 0.388498, 0.7582165, 0.49413198, ... \n", + "9 0 [0.6807515, 0.40140802, 0.75704247, 0.45774597... \n", + "10 0 [0.431925, 0.1701875, 0.561033, 0.3180745, 1.0... \n", + "11 0 [0.42018852, 0.34507048, 0.5434275, 0.4941315,... \n", + "12 0 [0.3732395, 0.348592, 0.54812247, 0.46009403, ... \n", + "13 0 [0.37793398, 0.3814555, 0.48122, 0.45892054, 1... \n", + "14 0 None \n", + " 1 [0.43427247, 0.3556335, 0.51173747, 0.43075046... \n", + " 2 [0.4929575, 0.44718304, 0.5375585, 0.485915, 1... \n", + "15 0 None \n", + " 1 [0.629108, 0.39788702, 0.67840403, 0.453051, 1... \n", + " 2 [0.4530515, 0.4107985, 0.58568054, 0.5105635, ... \n", + "16 0 None \n", + " 1 [0.45070451, 0.3967135, 0.5762915, 0.5152585, ... \n", + " 2 [0.63732404, 0.403756, 0.684272, 0.46009403, 1... \n", + "17 0 [0.43779355, 0.3826295, 0.61032856, 0.5223005,... \n", + "18 0 [0.4401405, 0.374413, 0.6068075, 0.529343, 1.0... \n", + "19 0 [0.536385, 0.4495305, 0.600939, 0.49999946, 0.... \n", + "20 0 None \n", + " 1 [0.52347445, 0.4424885, 0.6150235, 0.5281695, ... \n", + " 2 [0.456573, 0.515258, 0.504695, 0.562206, 0.0, ... \n", + " 3 [0.504695, 0.44248852, 0.54342705, 0.48004752,... \n", + "21 0 [0.54107946, 0.456573, 0.6103285, 0.511737, 0.... \n", + "22 0 [0.3298125, 0.2042255, 0.4225355, 0.2957745, 0... \n", + "23 0 [0.517606, 0.18661949, 0.615024, 0.2875585, 1.... \n", + "24 0 [0.51173747, 0.18779299, 0.6126765, 0.308685, ... \n", + "25 0 None \n", + " 1 [0.30985948, 0.3626765, 0.3861505, 0.43896753,... \n", + " 2 [0.41314548, 0.3427225, 0.48122054, 0.42018753... \n", + "26 0 [0.25821602, 0.30046952, 0.46830997, 0.4577465... \n", + "27 0 [0.2734745, 0.2828635, 0.44718352, 0.4577465, ... \n", + "28 0 [0.5915495, 0.23591551, 0.6455405, 0.2887325, ... \n", + "29 0 [0.5903755, 0.231221, 0.6420185, 0.280517, 1.0... \n", + "30 0 [0.40493003, 0.2018775, 0.469484, 0.2699525, 1... \n", + "31 0 [0.39554, 0.19483551, 0.483568, 0.2887325, 1.0... \n", + "32 0 [0.4025825, 0.212441, 0.45539945, 0.269953, 1.... \n", + "33 0 [0.2265255, 0.2136145, 0.3438965, 0.30751148, ... \n", + "34 0 [0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0] \n", + "35 0 [0.6079815, 0.320423, 0.7969485, 0.489437, 1.0... \n", + "36 0 [0.6185445, 0.30164298, 0.7758215, 0.494131, 1... \n", + "\n", + " discarded origin task_type last_seen group_id \\\n", + "sample_id annotation_id \n", + "0 0 False train_loader -1.0 0 \n", + "1 0 False train_loader -1.0 1 \n", + "2 0 False train_loader -1.0 2 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "3 0 False train_loader -1.0 3 \n", + "4 0 False train_loader -1.0 4 \n", + "5 0 False train_loader -1.0 5 \n", + "6 0 False train_loader -1.0 6 \n", + "7 0 False train_loader -1.0 7 \n", + "8 0 False train_loader -1.0 8 \n", + "9 0 False train_loader -1.0 9 \n", + "10 0 False train_loader -1.0 10 \n", + "11 0 False train_loader -1.0 11 \n", + "12 0 False train_loader -1.0 12 \n", + "13 0 False train_loader -1.0 13 \n", + "14 0 False train_loader -1.0 14 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "15 0 False train_loader -1.0 15 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "16 0 False train_loader -1.0 16 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "17 0 False train_loader -1.0 17 \n", + "18 0 False train_loader -1.0 18 \n", + "19 0 False train_loader -1.0 19 \n", + "20 0 False train_loader -1.0 20 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + " 3 None NaN None NaN None \n", + "21 0 False train_loader -1.0 21 \n", + "22 0 False train_loader -1.0 22 \n", + "23 0 False train_loader -1.0 23 \n", + "24 0 False train_loader -1.0 24 \n", + "25 0 False train_loader -1.0 25 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "26 0 False train_loader -1.0 26 \n", + "27 0 False train_loader -1.0 27 \n", + "28 0 False train_loader -1.0 28 \n", + "29 0 False train_loader -1.0 29 \n", + "30 0 False train_loader -1.0 30 \n", + "31 0 False train_loader -1.0 31 \n", + "32 0 False train_loader -1.0 32 \n", + "33 0 False train_loader -1.0 33 \n", + "34 0 False train_loader -1.0 34 \n", + "35 0 False train_loader -1.0 35 \n", + "36 0 False train_loader -1.0 36 \n", + "\n", + " member_rank \\\n", + "sample_id annotation_id \n", + "0 0 0.0 \n", + "1 0 0.0 \n", + "2 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "3 0 0.0 \n", + "4 0 0.0 \n", + "5 0 0.0 \n", + "6 0 0.0 \n", + "7 0 0.0 \n", + "8 0 0.0 \n", + "9 0 0.0 \n", + "10 0 0.0 \n", + "11 0 0.0 \n", + "12 0 0.0 \n", + "13 0 0.0 \n", + "14 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "15 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "16 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "17 0 0.0 \n", + "18 0 0.0 \n", + "19 0 0.0 \n", + "20 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + " 3 NaN \n", + "21 0 0.0 \n", + "22 0 0.0 \n", + "23 0 0.0 \n", + "24 0 0.0 \n", + "25 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "26 0 0.0 \n", + "27 0 0.0 \n", + "28 0 0.0 \n", + "29 0 0.0 \n", + "30 0 0.0 \n", + "31 0 0.0 \n", + "32 0 0.0 \n", + "33 0 0.0 \n", + "34 0 0.0 \n", + "35 0 0.0 \n", + "36 0 0.0 \n", + "\n", + " img_path \\\n", + "sample_id annotation_id \n", + "0 0 /content/datasets/brain-tumor/images/train/000... \n", + "1 0 /content/datasets/brain-tumor/images/train/000... \n", + "2 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "3 0 /content/datasets/brain-tumor/images/train/000... \n", + "4 0 /content/datasets/brain-tumor/images/train/000... \n", + "5 0 /content/datasets/brain-tumor/images/train/000... \n", + "6 0 /content/datasets/brain-tumor/images/train/000... \n", + "7 0 /content/datasets/brain-tumor/images/train/000... \n", + "8 0 /content/datasets/brain-tumor/images/train/000... \n", + "9 0 /content/datasets/brain-tumor/images/train/000... \n", + "10 0 /content/datasets/brain-tumor/images/train/000... \n", + "11 0 /content/datasets/brain-tumor/images/train/000... \n", + "12 0 /content/datasets/brain-tumor/images/train/000... \n", + "13 0 /content/datasets/brain-tumor/images/train/000... \n", + "14 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "15 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "16 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "17 0 /content/datasets/brain-tumor/images/train/000... \n", + "18 0 /content/datasets/brain-tumor/images/train/000... \n", + "19 0 /content/datasets/brain-tumor/images/train/000... \n", + "20 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 /content/datasets/brain-tumor/images/train/000... \n", + "22 0 /content/datasets/brain-tumor/images/train/000... \n", + "23 0 /content/datasets/brain-tumor/images/train/000... \n", + "24 0 /content/datasets/brain-tumor/images/train/000... \n", + "25 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "26 0 /content/datasets/brain-tumor/images/train/000... \n", + "27 0 /content/datasets/brain-tumor/images/train/000... \n", + "28 0 /content/datasets/brain-tumor/images/train/000... \n", + "29 0 /content/datasets/brain-tumor/images/train/000... \n", + "30 0 /content/datasets/brain-tumor/images/train/000... \n", + "31 0 /content/datasets/brain-tumor/images/train/000... \n", + "32 0 /content/datasets/brain-tumor/images/train/000... \n", + "33 0 /content/datasets/brain-tumor/images/train/000... \n", + "34 0 /content/datasets/brain-tumor/images/train/000... \n", + "35 0 /content/datasets/brain-tumor/images/train/000... \n", + "36 0 /content/datasets/brain-tumor/images/train/000... \n", + "\n", + " cls \n", + "sample_id annotation_id \n", + "0 0 [[1.0]] \n", + "1 0 [[1.0]] \n", + "2 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "3 0 [[1.0]] \n", + "4 0 [[1.0]] \n", + "5 0 [[1.0]] \n", + "6 0 [[1.0]] \n", + "7 0 [[1.0]] \n", + "8 0 [[1.0]] \n", + "9 0 [[1.0]] \n", + "10 0 [[1.0]] \n", + "11 0 [[1.0]] \n", + "12 0 [[1.0]] \n", + "13 0 [[1.0]] \n", + "14 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "15 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "16 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "17 0 [[1.0]] \n", + "18 0 [[1.0]] \n", + "19 0 [[0.0]] \n", + "20 0 [[0.0], [0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 [[0.0]] \n", + "22 0 [[0.0]] \n", + "23 0 [[1.0]] \n", + "24 0 [[1.0]] \n", + "25 0 [[0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + "26 0 [[0.0]] \n", + "27 0 [[0.0]] \n", + "28 0 [[1.0]] \n", + "29 0 [[1.0]] \n", + "30 0 [[1.0]] \n", + "31 0 [[1.0]] \n", + "32 0 [[1.0]] \n", + "33 0 [[1.0]] \n", + "34 0 [[1.0]] \n", + "35 0 [[1.0]] \n", + "36 0 [[1.0]] " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
predictionprediction_rawtargetdiscardedorigintask_typelast_seengroup_idmember_rankimg_pathcls
sample_idannotation_id
00NoneNone[0.2335685, 0.254695, 0.4553995, 0.43075103, 1...Falsetrain_loader-1.000.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
10NoneNone[0.251174, 0.24061048, 0.44366202, 0.4307515, ...Falsetrain_loader-1.010.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
20NoneNoneNoneFalsetrain_loader-1.020.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.523474, 0.3978875, 0.634976, 0.48122054, 1....NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.40610352, 0.415493, 0.5234745, 0.522301, 1....NoneNaNNoneNaNNoneNaNNoneNone
30NoneNone[0.403756, 0.3826295, 0.637324, 0.5152585, 1.0...Falsetrain_loader-1.030.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
40NoneNone[0.4436615, 0.3814555, 0.5938965, 0.4518785, 1...Falsetrain_loader-1.040.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
50NoneNone[0.6044605, 0.3990615, 0.67253554, 0.46596247,...Falsetrain_loader-1.050.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
60NoneNone[0.410798, 0.4436625, 0.556338, 0.51173747, 1....Falsetrain_loader-1.060.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
70NoneNone[0.650235, 0.4166665, 0.73826295, 0.48356748, ...Falsetrain_loader-1.070.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
80NoneNone[0.64788747, 0.388498, 0.7582165, 0.49413198, ...Falsetrain_loader-1.080.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
90NoneNone[0.6807515, 0.40140802, 0.75704247, 0.45774597...Falsetrain_loader-1.090.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
100NoneNone[0.431925, 0.1701875, 0.561033, 0.3180745, 1.0...Falsetrain_loader-1.0100.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
110NoneNone[0.42018852, 0.34507048, 0.5434275, 0.4941315,...Falsetrain_loader-1.0110.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
120NoneNone[0.3732395, 0.348592, 0.54812247, 0.46009403, ...Falsetrain_loader-1.0120.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
130NoneNone[0.37793398, 0.3814555, 0.48122, 0.45892054, 1...Falsetrain_loader-1.0130.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
140NoneNoneNoneFalsetrain_loader-1.0140.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.43427247, 0.3556335, 0.51173747, 0.43075046...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4929575, 0.44718304, 0.5375585, 0.485915, 1...NoneNaNNoneNaNNoneNaNNoneNone
150NoneNoneNoneFalsetrain_loader-1.0150.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.629108, 0.39788702, 0.67840403, 0.453051, 1...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4530515, 0.4107985, 0.58568054, 0.5105635, ...NoneNaNNoneNaNNoneNaNNoneNone
160NoneNoneNoneFalsetrain_loader-1.0160.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.45070451, 0.3967135, 0.5762915, 0.5152585, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.63732404, 0.403756, 0.684272, 0.46009403, 1...NoneNaNNoneNaNNoneNaNNoneNone
170NoneNone[0.43779355, 0.3826295, 0.61032856, 0.5223005,...Falsetrain_loader-1.0170.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
180NoneNone[0.4401405, 0.374413, 0.6068075, 0.529343, 1.0...Falsetrain_loader-1.0180.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
190NoneNone[0.536385, 0.4495305, 0.600939, 0.49999946, 0....Falsetrain_loader-1.0190.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
200NoneNoneNoneFalsetrain_loader-1.0200.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0], [0.0]]
1NoneNone[0.52347445, 0.4424885, 0.6150235, 0.5281695, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.456573, 0.515258, 0.504695, 0.562206, 0.0, ...NoneNaNNoneNaNNoneNaNNoneNone
3NoneNone[0.504695, 0.44248852, 0.54342705, 0.48004752,...NoneNaNNoneNaNNoneNaNNoneNone
210NoneNone[0.54107946, 0.456573, 0.6103285, 0.511737, 0....Falsetrain_loader-1.0210.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
220NoneNone[0.3298125, 0.2042255, 0.4225355, 0.2957745, 0...Falsetrain_loader-1.0220.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
230NoneNone[0.517606, 0.18661949, 0.615024, 0.2875585, 1....Falsetrain_loader-1.0230.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
240NoneNone[0.51173747, 0.18779299, 0.6126765, 0.308685, ...Falsetrain_loader-1.0240.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
250NoneNoneNoneFalsetrain_loader-1.0250.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0]]
1NoneNone[0.30985948, 0.3626765, 0.3861505, 0.43896753,...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.41314548, 0.3427225, 0.48122054, 0.42018753...NoneNaNNoneNaNNoneNaNNoneNone
260NoneNone[0.25821602, 0.30046952, 0.46830997, 0.4577465...Falsetrain_loader-1.0260.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
270NoneNone[0.2734745, 0.2828635, 0.44718352, 0.4577465, ...Falsetrain_loader-1.0270.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
280NoneNone[0.5915495, 0.23591551, 0.6455405, 0.2887325, ...Falsetrain_loader-1.0280.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
290NoneNone[0.5903755, 0.231221, 0.6420185, 0.280517, 1.0...Falsetrain_loader-1.0290.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
300NoneNone[0.40493003, 0.2018775, 0.469484, 0.2699525, 1...Falsetrain_loader-1.0300.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
310NoneNone[0.39554, 0.19483551, 0.483568, 0.2887325, 1.0...Falsetrain_loader-1.0310.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
320NoneNone[0.4025825, 0.212441, 0.45539945, 0.269953, 1....Falsetrain_loader-1.0320.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
330NoneNone[0.2265255, 0.2136145, 0.3438965, 0.30751148, ...Falsetrain_loader-1.0330.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
340NoneNone[0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0]Falsetrain_loader-1.0340.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
350NoneNone[0.6079815, 0.320423, 0.7969485, 0.489437, 1.0...Falsetrain_loader-1.0350.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
360NoneNone[0.6185445, 0.30164298, 0.7758215, 0.494131, 1...Falsetrain_loader-1.0360.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "repr_error": "Out of range float values are not JSON compliant: nan" + } + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "🔗 Open the full grid in Weights Studio" + ] + }, + "metadata": {} + } + ], + "source": [ + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "from weightslab.backend.ledgers import get_dataframe\n", + "\n", + "# `get_df_view()` is the same per-sample DataFrame that powers Studio's grid.\n", + "dfm = get_dataframe()\n", + "grid = dfm.get_df_view() if hasattr(dfm, \"get_df_view\") else pd.DataFrame()\n", + "print(f\"{len(grid)} samples tracked\")\n", + "display(grid.head(50))\n", + "\n", + "# Open the live, interactive grid (with image + bbox previews) in Weights Studio.\n", + "# Studio must be running locally (`weightslab start`).\n", + "display(HTML(\n", + " '🔗 Open the full grid in Weights Studio'\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FXQ0gBOSKs7t" + }, + "source": [ + "> **Can clicking a grid row here jump to that sample in Studio?** Not out of the box today. The notebook and Studio are separate processes, and the link above opens Studio at its root. Per-sample deep-linking (e.g. `http://localhost:5173/?sample=` selecting and scrolling to that row) is a small frontend addition to Weights Studio — the sample `uid` is already the grid's index here, so the notebook side is ready for it once Studio accepts the query param." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vlHp09Nueb3d" + }, + "source": [ + "## 💡Citation\n", + "\n", + "```bibtex\n", + "@dataset{Jocher_Ultralytics_Datasets_2025,\n", + " author = {Jocher, Glenn and Rizwan, Muhammad},\n", + " license = {AGPL-3.0},\n", + " month = {May},\n", + " title = {Ultralytics Datasets: HomeObjects-3K Detection Dataset},\n", + " url = {https://docs.ultralytics.com/datasets/detect/homeobject-3k/},\n", + " version = {1.0.0},\n", + " year = {2025}\n", + "}\n", + "```\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "A100", + "machine_shape": "hm", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-kitti-detection-dataset.ipynb b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-kitti-detection-dataset.ipynb new file mode 100644 index 00000000..d7443915 --- /dev/null +++ b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-kitti-detection-dataset.ipynb @@ -0,0 +1,1707 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Train an Ultralytics YOLO detector on the KITTI dataset and stream every sample, prediction and metric live into Weights Studio to inspect, edit and evolve your run.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# KITTI Detection · Ultralytics YOLO + WeightsLab\n", + "\n", + "This notebook trains a YOLO detector on the [KITTI](https://docs.ultralytics.com/) dataset (detecting cars, pedestrians and cyclists in autonomous-driving street scenes) **through the WeightsLab training pipeline**.\n", + "\n", + "Instead of Ultralytics' one-shot `train`/`predict` flow, we hand training to `WLAwareTrainer`. It wraps the train/val dataloaders, logs a **per-sample** signal for every image (loss, prediction stats, tags) and ships live prediction overlays to **Weights Studio** — where you inspect the data grid, remove/weight samples, edit the model and resume, all while training runs.\n", + "\n", + "Ultralytics auto-downloads the dataset the first time you train." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o7YFKEN1Ks7Y" + }, + "source": [ + "## 1 · Setup\n", + "\n", + "Install WeightsLab (which pulls in Ultralytics) and verify the environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "71be394c", + "outputId": "c9ef4767-84f5-4bea-e395-9e11766002f2", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "source": [ + "%pip install setuptools wheel" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (75.2.0)\n", + "Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (0.47.0)\n", + "Requirement already satisfied: packaging>=24.0 in /usr/local/lib/python3.12/dist-packages (from wheel) (26.2)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "%pip install git+https://github.com/GrayboxTech/weightslab.git" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "jV1oi_PQN0xK", + "outputId": "eda8eca1-f4b3-4ffd-eaa6-0f67ce857153" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting git+https://github.com/GrayboxTech/weightslab.git\n", + " Cloning https://github.com/GrayboxTech/weightslab.git (to revision landingcollab) to /tmp/pip-req-build-z4ba7cx9\n", + " Running command git clone --filter=blob:none --quiet https://github.com/GrayboxTech/weightslab.git /tmp/pip-req-build-z4ba7cx9\n", + " Running command git checkout -b landingcollab --track origin/landingcollab\n", + " Switched to a new branch 'landingcollab'\n", + " Branch 'landingcollab' set up to track remote branch 'landingcollab' from 'origin'.\n", + " Resolved https://github.com/GrayboxTech/weightslab.git to commit 3f1cf143bc93e67df43adc71ab831fa71477e0d2\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.10.2)\n", + "Requirement already satisfied: torch<=2.9,>=2.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.9.0)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.24.0)\n", + "Requirement already satisfied: torchmetrics>=1.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.21)\n", + "Requirement already satisfied: onnx<=1.20,>=1.15 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.20.0)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.4.9)\n", + "Requirement already satisfied: langchain-ollama<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.1.0)\n", + "Requirement already satisfied: langchain-openai<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.5)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev8) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.2)\n", + "Requirement already satisfied: ollama<1.0.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev8) (0.6.2)\n", + "Requirement already satisfied: openai<3.0.0,>=2.45.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2.45.0)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev8) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.5.0)\n", + "Requirement already satisfied: lightning-utilities>=0.15.3 in /usr/local/lib/python3.12/dist-packages (from torchmetrics>=1.9->weightslab==1.3.3.dev8) (0.15.3)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.0.0)\n", + "Building wheels for collected packages: weightslab\n", + " Building wheel for weightslab (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for weightslab: filename=weightslab-1.3.3.dev8-py3-none-any.whl size=2827715 sha256=5c4e7c14bbf95a48b9a8c86b5a8c744af67e1eb395197cbe79d3e443c21cf86e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-__zwgf7x/wheels/4d/c7/cd/817c2745790555e7b6c532f5b6055d47567f9416fdfc65879b\n", + "Successfully built weightslab\n", + "Installing collected packages: weightslab\n", + " Attempting uninstall: weightslab\n", + " Found existing installation: weightslab 1.3.3.dev7\n", + " Uninstalling weightslab-1.3.3.dev7:\n", + " Successfully uninstalled weightslab-1.3.3.dev7\n", + "Successfully installed weightslab-1.3.3.dev8\n" + ] + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "weightslab" + ] + }, + "id": "b7fce274042740d19087972fe9c01c9b" + } + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pJhvREbaKs7f", + "outputId": "651baa36-9ebb-49da-830b-e6c8a6e2b64f" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Ultralytics 8.4.96 🚀 Python-3.12.13 torch-2.9.0+cu128 CPU (Intel Xeon CPU @ 2.20GHz)\n", + "Setup complete ✅ (2 CPUs, 12.7 GB RAM, 30.5/107.7 GB disk)\n", + "16/07/2026-09:29:29.683 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmp6meuzqdz/weightslab_logs/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:29.685 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-09:29:29.686 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev8 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-09:29:29.690 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install weightslab\n", + "# %pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev8\"\n", + "%pip install ultralytics\n", + "\n", + "import ultralytics\n", + "ultralytics.checks()\n", + "\n", + "import weightslab as wl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 · Dataset\n", + "\n", + "The dataset is described by a small YAML file that Ultralytics resolves and downloads automatically. For reference:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h8go3HNgN0WU" + }, + "source": [ + "```yaml\n", + "# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license\n", + "\n", + "# Kitti dataset by Karlsruhe Institute of Technology and Toyota Technological Institute at Chicago\n", + "# Documentation: https://docs.ultralytics.com/datasets/detect/kitti/\n", + "# Example usage: yolo train data=kitti.yaml\n", + "# parent\n", + "# ├── ultralytics\n", + "# └── datasets\n", + "# └── kitti ← downloads here (390.5 MB)\n", + "\n", + "# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..]\n", + "path: kitti # dataset root dir\n", + "train: images/train # train images (relative to 'path') 5985 images\n", + "val: images/val # val images (relative to 'path') 1496 images\n", + "\n", + "names:\n", + " 0: car\n", + " 1: van\n", + " 2: truck\n", + " 3: pedestrian\n", + " 4: person_sitting\n", + " 5: cyclist\n", + " 6: tram\n", + " 7: misc\n", + "\n", + "# Download script/URL (optional)\n", + "download: https://github.com/ultralytics/assets/releases/download/v0.0.0/kitti.zip\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rH79t30YKs7i" + }, + "source": [ + "## 3 · Train through WeightsLab\n", + "\n", + "We register the run's hyperparameters with `wl.watch_or_edit(...)` (so they become live-editable from Studio), start the backend services with `wl.serve()`, then hand training to `WLAwareTrainer` via the standard `YOLO(...).train(trainer=...)` entry point.\n", + "\n", + "> **Connect Weights Studio** — run `weightslab start` locally (opens `http://localhost:5173`) *before or during* training to watch the run live. On **Colab**, serve over a public tunnel instead: use `wl.serve(serving_grpc=True, serving_bore=True)` and connect your local Studio with `weightslab tunnel bore.pub:PORT` (the port is printed when serving starts).\n", + "\n", + "Data augmentations are disabled so each sample maps cleanly to its ground truth in the grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ.setdefault(\"WL_PRELOAD_IMAGE_OVERVIEW\", \"0\")\n", + "os.environ.setdefault(\"WEIGHTSLAB_LOG_LEVEL\", \"WARNING\")\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "from weightslab.integrations.ultralytics import WLAwareTrainer\n", + "from ultralytics import YOLO\n", + "\n", + "# Hyperparameters registered with WeightsLab -> live-editable from Weights Studio.\n", + "cfg = {\n", + " \"model\": \"yolo11n.pt\", # pretrained checkpoint\n", + " \"data\": \"kitti.yaml\", # Ultralytics auto-downloads\n", + " \"image_size\": 640,\n", + " \"epochs\": 10,\n", + " # Per-sample prediction overlays streamed to Studio (NMS on the train set).\n", + " \"signals_cfg\": {\n", + " \"train_nms\": {\"conf_thres\": 0.25, \"iou_thres\": 0.45, \"max_nms\": 7},\n", + " },\n", + "}\n", + "wl.watch_or_edit(cfg, flag=\"hyperparameters\", defaults=cfg, poll_interval=1.0)\n", + "\n", + "# Start the WeightsLab backend services that Weights Studio connects to.\n", + "wl.serve(serving_grpc=True)\n", + "# Colab -> local Studio: wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "wl.start_training() # equivalent to pressing \"play\" in Studio\n", + "\n", + "# Read back the (now live) hyperparameters and hand training to WLAwareTrainer.\n", + "model = YOLO(cfg[\"model\"])\n", + "results = model.train(\n", + " trainer=WLAwareTrainer,\n", + " data=str(cfg[\"data\"]),\n", + " imgsz=cfg[\"image_size\"],\n", + " epochs=int(cfg[\"epochs\"]),\n", + " project=\"./wl_logs\", name=\"kitti\", # -> WL log_dir/name\n", + " workers=0, # WL invariant (parent-process uid counter)\n", + " optimizer=\"SGD\", lr0=0.001, amp=False,\n", + " # All augmentations off for a clean sample <-> ground-truth association.\n", + " mosaic=0.0, mixup=0.0, copy_paste=0.0,\n", + " hsv_h=0.0, hsv_s=0.0, hsv_v=0.0,\n", + " degrees=0.0, translate=0.0, scale=0.0, shear=0.0, perspective=0.0,\n", + " flipud=0.0, fliplr=0.0, erasing=0.0, auto_augment=None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f-Uw_s_8Ks7m" + }, + "source": [ + "## 4 · Explore the data grid\n", + "\n", + "The table below is exactly the data behind Weights Studio's **grid explorer**: one row per sample, keyed by split (`origin`) and sample id, with the per-sample loss, prediction stats and tags accumulated during training. This is the ledger that Studio reads from — here we render it inline with pandas.\n", + "\n", + "The image thumbnails and bounding-box overlays themselves are rendered by **Weights Studio** (which streams full previews over gRPC); the link below opens the interactive grid there." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "collapsed": true, + "id": "uOJdXGA3Ks7o", + "outputId": "26652fa3-6a15-43ce-9d75-034eaba21ccd" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "1241 samples tracked\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " prediction prediction_raw \\\n", + "sample_id annotation_id \n", + "0 0 None None \n", + "1 0 None None \n", + "2 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "3 0 None None \n", + "4 0 None None \n", + "5 0 None None \n", + "6 0 None None \n", + "7 0 None None \n", + "8 0 None None \n", + "9 0 None None \n", + "10 0 None None \n", + "11 0 None None \n", + "12 0 None None \n", + "13 0 None None \n", + "14 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "15 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "16 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "17 0 None None \n", + "18 0 None None \n", + "19 0 None None \n", + "20 0 None None \n", + " 1 None None \n", + " 2 None None \n", + " 3 None None \n", + "21 0 None None \n", + "22 0 None None \n", + "23 0 None None \n", + "24 0 None None \n", + "25 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "26 0 None None \n", + "27 0 None None \n", + "28 0 None None \n", + "29 0 None None \n", + "30 0 None None \n", + "31 0 None None \n", + "32 0 None None \n", + "33 0 None None \n", + "34 0 None None \n", + "35 0 None None \n", + "36 0 None None \n", + "\n", + " target \\\n", + "sample_id annotation_id \n", + "0 0 [0.2335685, 0.254695, 0.4553995, 0.43075103, 1... \n", + "1 0 [0.251174, 0.24061048, 0.44366202, 0.4307515, ... \n", + "2 0 None \n", + " 1 [0.523474, 0.3978875, 0.634976, 0.48122054, 1.... \n", + " 2 [0.40610352, 0.415493, 0.5234745, 0.522301, 1.... \n", + "3 0 [0.403756, 0.3826295, 0.637324, 0.5152585, 1.0... \n", + "4 0 [0.4436615, 0.3814555, 0.5938965, 0.4518785, 1... \n", + "5 0 [0.6044605, 0.3990615, 0.67253554, 0.46596247,... \n", + "6 0 [0.410798, 0.4436625, 0.556338, 0.51173747, 1.... \n", + "7 0 [0.650235, 0.4166665, 0.73826295, 0.48356748, ... \n", + "8 0 [0.64788747, 0.388498, 0.7582165, 0.49413198, ... \n", + "9 0 [0.6807515, 0.40140802, 0.75704247, 0.45774597... \n", + "10 0 [0.431925, 0.1701875, 0.561033, 0.3180745, 1.0... \n", + "11 0 [0.42018852, 0.34507048, 0.5434275, 0.4941315,... \n", + "12 0 [0.3732395, 0.348592, 0.54812247, 0.46009403, ... \n", + "13 0 [0.37793398, 0.3814555, 0.48122, 0.45892054, 1... \n", + "14 0 None \n", + " 1 [0.43427247, 0.3556335, 0.51173747, 0.43075046... \n", + " 2 [0.4929575, 0.44718304, 0.5375585, 0.485915, 1... \n", + "15 0 None \n", + " 1 [0.629108, 0.39788702, 0.67840403, 0.453051, 1... \n", + " 2 [0.4530515, 0.4107985, 0.58568054, 0.5105635, ... \n", + "16 0 None \n", + " 1 [0.45070451, 0.3967135, 0.5762915, 0.5152585, ... \n", + " 2 [0.63732404, 0.403756, 0.684272, 0.46009403, 1... \n", + "17 0 [0.43779355, 0.3826295, 0.61032856, 0.5223005,... \n", + "18 0 [0.4401405, 0.374413, 0.6068075, 0.529343, 1.0... \n", + "19 0 [0.536385, 0.4495305, 0.600939, 0.49999946, 0.... \n", + "20 0 None \n", + " 1 [0.52347445, 0.4424885, 0.6150235, 0.5281695, ... \n", + " 2 [0.456573, 0.515258, 0.504695, 0.562206, 0.0, ... \n", + " 3 [0.504695, 0.44248852, 0.54342705, 0.48004752,... \n", + "21 0 [0.54107946, 0.456573, 0.6103285, 0.511737, 0.... \n", + "22 0 [0.3298125, 0.2042255, 0.4225355, 0.2957745, 0... \n", + "23 0 [0.517606, 0.18661949, 0.615024, 0.2875585, 1.... \n", + "24 0 [0.51173747, 0.18779299, 0.6126765, 0.308685, ... \n", + "25 0 None \n", + " 1 [0.30985948, 0.3626765, 0.3861505, 0.43896753,... \n", + " 2 [0.41314548, 0.3427225, 0.48122054, 0.42018753... \n", + "26 0 [0.25821602, 0.30046952, 0.46830997, 0.4577465... \n", + "27 0 [0.2734745, 0.2828635, 0.44718352, 0.4577465, ... \n", + "28 0 [0.5915495, 0.23591551, 0.6455405, 0.2887325, ... \n", + "29 0 [0.5903755, 0.231221, 0.6420185, 0.280517, 1.0... \n", + "30 0 [0.40493003, 0.2018775, 0.469484, 0.2699525, 1... \n", + "31 0 [0.39554, 0.19483551, 0.483568, 0.2887325, 1.0... \n", + "32 0 [0.4025825, 0.212441, 0.45539945, 0.269953, 1.... \n", + "33 0 [0.2265255, 0.2136145, 0.3438965, 0.30751148, ... \n", + "34 0 [0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0] \n", + "35 0 [0.6079815, 0.320423, 0.7969485, 0.489437, 1.0... \n", + "36 0 [0.6185445, 0.30164298, 0.7758215, 0.494131, 1... \n", + "\n", + " discarded origin task_type last_seen group_id \\\n", + "sample_id annotation_id \n", + "0 0 False train_loader -1.0 0 \n", + "1 0 False train_loader -1.0 1 \n", + "2 0 False train_loader -1.0 2 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "3 0 False train_loader -1.0 3 \n", + "4 0 False train_loader -1.0 4 \n", + "5 0 False train_loader -1.0 5 \n", + "6 0 False train_loader -1.0 6 \n", + "7 0 False train_loader -1.0 7 \n", + "8 0 False train_loader -1.0 8 \n", + "9 0 False train_loader -1.0 9 \n", + "10 0 False train_loader -1.0 10 \n", + "11 0 False train_loader -1.0 11 \n", + "12 0 False train_loader -1.0 12 \n", + "13 0 False train_loader -1.0 13 \n", + "14 0 False train_loader -1.0 14 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "15 0 False train_loader -1.0 15 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "16 0 False train_loader -1.0 16 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "17 0 False train_loader -1.0 17 \n", + "18 0 False train_loader -1.0 18 \n", + "19 0 False train_loader -1.0 19 \n", + "20 0 False train_loader -1.0 20 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + " 3 None NaN None NaN None \n", + "21 0 False train_loader -1.0 21 \n", + "22 0 False train_loader -1.0 22 \n", + "23 0 False train_loader -1.0 23 \n", + "24 0 False train_loader -1.0 24 \n", + "25 0 False train_loader -1.0 25 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "26 0 False train_loader -1.0 26 \n", + "27 0 False train_loader -1.0 27 \n", + "28 0 False train_loader -1.0 28 \n", + "29 0 False train_loader -1.0 29 \n", + "30 0 False train_loader -1.0 30 \n", + "31 0 False train_loader -1.0 31 \n", + "32 0 False train_loader -1.0 32 \n", + "33 0 False train_loader -1.0 33 \n", + "34 0 False train_loader -1.0 34 \n", + "35 0 False train_loader -1.0 35 \n", + "36 0 False train_loader -1.0 36 \n", + "\n", + " member_rank \\\n", + "sample_id annotation_id \n", + "0 0 0.0 \n", + "1 0 0.0 \n", + "2 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "3 0 0.0 \n", + "4 0 0.0 \n", + "5 0 0.0 \n", + "6 0 0.0 \n", + "7 0 0.0 \n", + "8 0 0.0 \n", + "9 0 0.0 \n", + "10 0 0.0 \n", + "11 0 0.0 \n", + "12 0 0.0 \n", + "13 0 0.0 \n", + "14 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "15 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "16 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "17 0 0.0 \n", + "18 0 0.0 \n", + "19 0 0.0 \n", + "20 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + " 3 NaN \n", + "21 0 0.0 \n", + "22 0 0.0 \n", + "23 0 0.0 \n", + "24 0 0.0 \n", + "25 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "26 0 0.0 \n", + "27 0 0.0 \n", + "28 0 0.0 \n", + "29 0 0.0 \n", + "30 0 0.0 \n", + "31 0 0.0 \n", + "32 0 0.0 \n", + "33 0 0.0 \n", + "34 0 0.0 \n", + "35 0 0.0 \n", + "36 0 0.0 \n", + "\n", + " img_path \\\n", + "sample_id annotation_id \n", + "0 0 /content/datasets/brain-tumor/images/train/000... \n", + "1 0 /content/datasets/brain-tumor/images/train/000... \n", + "2 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "3 0 /content/datasets/brain-tumor/images/train/000... \n", + "4 0 /content/datasets/brain-tumor/images/train/000... \n", + "5 0 /content/datasets/brain-tumor/images/train/000... \n", + "6 0 /content/datasets/brain-tumor/images/train/000... \n", + "7 0 /content/datasets/brain-tumor/images/train/000... \n", + "8 0 /content/datasets/brain-tumor/images/train/000... \n", + "9 0 /content/datasets/brain-tumor/images/train/000... \n", + "10 0 /content/datasets/brain-tumor/images/train/000... \n", + "11 0 /content/datasets/brain-tumor/images/train/000... \n", + "12 0 /content/datasets/brain-tumor/images/train/000... \n", + "13 0 /content/datasets/brain-tumor/images/train/000... \n", + "14 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "15 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "16 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "17 0 /content/datasets/brain-tumor/images/train/000... \n", + "18 0 /content/datasets/brain-tumor/images/train/000... \n", + "19 0 /content/datasets/brain-tumor/images/train/000... \n", + "20 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 /content/datasets/brain-tumor/images/train/000... \n", + "22 0 /content/datasets/brain-tumor/images/train/000... \n", + "23 0 /content/datasets/brain-tumor/images/train/000... \n", + "24 0 /content/datasets/brain-tumor/images/train/000... \n", + "25 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "26 0 /content/datasets/brain-tumor/images/train/000... \n", + "27 0 /content/datasets/brain-tumor/images/train/000... \n", + "28 0 /content/datasets/brain-tumor/images/train/000... \n", + "29 0 /content/datasets/brain-tumor/images/train/000... \n", + "30 0 /content/datasets/brain-tumor/images/train/000... \n", + "31 0 /content/datasets/brain-tumor/images/train/000... \n", + "32 0 /content/datasets/brain-tumor/images/train/000... \n", + "33 0 /content/datasets/brain-tumor/images/train/000... \n", + "34 0 /content/datasets/brain-tumor/images/train/000... \n", + "35 0 /content/datasets/brain-tumor/images/train/000... \n", + "36 0 /content/datasets/brain-tumor/images/train/000... \n", + "\n", + " cls \n", + "sample_id annotation_id \n", + "0 0 [[1.0]] \n", + "1 0 [[1.0]] \n", + "2 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "3 0 [[1.0]] \n", + "4 0 [[1.0]] \n", + "5 0 [[1.0]] \n", + "6 0 [[1.0]] \n", + "7 0 [[1.0]] \n", + "8 0 [[1.0]] \n", + "9 0 [[1.0]] \n", + "10 0 [[1.0]] \n", + "11 0 [[1.0]] \n", + "12 0 [[1.0]] \n", + "13 0 [[1.0]] \n", + "14 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "15 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "16 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "17 0 [[1.0]] \n", + "18 0 [[1.0]] \n", + "19 0 [[0.0]] \n", + "20 0 [[0.0], [0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 [[0.0]] \n", + "22 0 [[0.0]] \n", + "23 0 [[1.0]] \n", + "24 0 [[1.0]] \n", + "25 0 [[0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + "26 0 [[0.0]] \n", + "27 0 [[0.0]] \n", + "28 0 [[1.0]] \n", + "29 0 [[1.0]] \n", + "30 0 [[1.0]] \n", + "31 0 [[1.0]] \n", + "32 0 [[1.0]] \n", + "33 0 [[1.0]] \n", + "34 0 [[1.0]] \n", + "35 0 [[1.0]] \n", + "36 0 [[1.0]] " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
predictionprediction_rawtargetdiscardedorigintask_typelast_seengroup_idmember_rankimg_pathcls
sample_idannotation_id
00NoneNone[0.2335685, 0.254695, 0.4553995, 0.43075103, 1...Falsetrain_loader-1.000.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
10NoneNone[0.251174, 0.24061048, 0.44366202, 0.4307515, ...Falsetrain_loader-1.010.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
20NoneNoneNoneFalsetrain_loader-1.020.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.523474, 0.3978875, 0.634976, 0.48122054, 1....NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.40610352, 0.415493, 0.5234745, 0.522301, 1....NoneNaNNoneNaNNoneNaNNoneNone
30NoneNone[0.403756, 0.3826295, 0.637324, 0.5152585, 1.0...Falsetrain_loader-1.030.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
40NoneNone[0.4436615, 0.3814555, 0.5938965, 0.4518785, 1...Falsetrain_loader-1.040.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
50NoneNone[0.6044605, 0.3990615, 0.67253554, 0.46596247,...Falsetrain_loader-1.050.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
60NoneNone[0.410798, 0.4436625, 0.556338, 0.51173747, 1....Falsetrain_loader-1.060.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
70NoneNone[0.650235, 0.4166665, 0.73826295, 0.48356748, ...Falsetrain_loader-1.070.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
80NoneNone[0.64788747, 0.388498, 0.7582165, 0.49413198, ...Falsetrain_loader-1.080.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
90NoneNone[0.6807515, 0.40140802, 0.75704247, 0.45774597...Falsetrain_loader-1.090.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
100NoneNone[0.431925, 0.1701875, 0.561033, 0.3180745, 1.0...Falsetrain_loader-1.0100.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
110NoneNone[0.42018852, 0.34507048, 0.5434275, 0.4941315,...Falsetrain_loader-1.0110.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
120NoneNone[0.3732395, 0.348592, 0.54812247, 0.46009403, ...Falsetrain_loader-1.0120.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
130NoneNone[0.37793398, 0.3814555, 0.48122, 0.45892054, 1...Falsetrain_loader-1.0130.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
140NoneNoneNoneFalsetrain_loader-1.0140.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.43427247, 0.3556335, 0.51173747, 0.43075046...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4929575, 0.44718304, 0.5375585, 0.485915, 1...NoneNaNNoneNaNNoneNaNNoneNone
150NoneNoneNoneFalsetrain_loader-1.0150.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.629108, 0.39788702, 0.67840403, 0.453051, 1...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4530515, 0.4107985, 0.58568054, 0.5105635, ...NoneNaNNoneNaNNoneNaNNoneNone
160NoneNoneNoneFalsetrain_loader-1.0160.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.45070451, 0.3967135, 0.5762915, 0.5152585, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.63732404, 0.403756, 0.684272, 0.46009403, 1...NoneNaNNoneNaNNoneNaNNoneNone
170NoneNone[0.43779355, 0.3826295, 0.61032856, 0.5223005,...Falsetrain_loader-1.0170.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
180NoneNone[0.4401405, 0.374413, 0.6068075, 0.529343, 1.0...Falsetrain_loader-1.0180.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
190NoneNone[0.536385, 0.4495305, 0.600939, 0.49999946, 0....Falsetrain_loader-1.0190.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
200NoneNoneNoneFalsetrain_loader-1.0200.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0], [0.0]]
1NoneNone[0.52347445, 0.4424885, 0.6150235, 0.5281695, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.456573, 0.515258, 0.504695, 0.562206, 0.0, ...NoneNaNNoneNaNNoneNaNNoneNone
3NoneNone[0.504695, 0.44248852, 0.54342705, 0.48004752,...NoneNaNNoneNaNNoneNaNNoneNone
210NoneNone[0.54107946, 0.456573, 0.6103285, 0.511737, 0....Falsetrain_loader-1.0210.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
220NoneNone[0.3298125, 0.2042255, 0.4225355, 0.2957745, 0...Falsetrain_loader-1.0220.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
230NoneNone[0.517606, 0.18661949, 0.615024, 0.2875585, 1....Falsetrain_loader-1.0230.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
240NoneNone[0.51173747, 0.18779299, 0.6126765, 0.308685, ...Falsetrain_loader-1.0240.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
250NoneNoneNoneFalsetrain_loader-1.0250.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0]]
1NoneNone[0.30985948, 0.3626765, 0.3861505, 0.43896753,...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.41314548, 0.3427225, 0.48122054, 0.42018753...NoneNaNNoneNaNNoneNaNNoneNone
260NoneNone[0.25821602, 0.30046952, 0.46830997, 0.4577465...Falsetrain_loader-1.0260.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
270NoneNone[0.2734745, 0.2828635, 0.44718352, 0.4577465, ...Falsetrain_loader-1.0270.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
280NoneNone[0.5915495, 0.23591551, 0.6455405, 0.2887325, ...Falsetrain_loader-1.0280.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
290NoneNone[0.5903755, 0.231221, 0.6420185, 0.280517, 1.0...Falsetrain_loader-1.0290.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
300NoneNone[0.40493003, 0.2018775, 0.469484, 0.2699525, 1...Falsetrain_loader-1.0300.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
310NoneNone[0.39554, 0.19483551, 0.483568, 0.2887325, 1.0...Falsetrain_loader-1.0310.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
320NoneNone[0.4025825, 0.212441, 0.45539945, 0.269953, 1....Falsetrain_loader-1.0320.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
330NoneNone[0.2265255, 0.2136145, 0.3438965, 0.30751148, ...Falsetrain_loader-1.0330.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
340NoneNone[0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0]Falsetrain_loader-1.0340.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
350NoneNone[0.6079815, 0.320423, 0.7969485, 0.489437, 1.0...Falsetrain_loader-1.0350.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
360NoneNone[0.6185445, 0.30164298, 0.7758215, 0.494131, 1...Falsetrain_loader-1.0360.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "repr_error": "Out of range float values are not JSON compliant: nan" + } + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "🔗 Open the full grid in Weights Studio" + ] + }, + "metadata": {} + } + ], + "source": [ + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "from weightslab.backend.ledgers import get_dataframe\n", + "\n", + "# `get_df_view()` is the same per-sample DataFrame that powers Studio's grid.\n", + "dfm = get_dataframe()\n", + "grid = dfm.get_df_view() if hasattr(dfm, \"get_df_view\") else pd.DataFrame()\n", + "print(f\"{len(grid)} samples tracked\")\n", + "display(grid.head(50))\n", + "\n", + "# Open the live, interactive grid (with image + bbox previews) in Weights Studio.\n", + "# Studio must be running locally (`weightslab start`).\n", + "display(HTML(\n", + " '🔗 Open the full grid in Weights Studio'\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FXQ0gBOSKs7t" + }, + "source": [ + "> **Can clicking a grid row here jump to that sample in Studio?** Not out of the box today. The notebook and Studio are separate processes, and the link above opens Studio at its root. Per-sample deep-linking (e.g. `http://localhost:5173/?sample=` selecting and scrolling to that row) is a small frontend addition to Weights Studio — the sample `uid` is already the grid's index here, so the notebook side is ready for it once Studio accepts the query param." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vlHp09Nueb3d" + }, + "source": [ + "## Citation,License and Attribution\n", + "\n", + "```bibtex\n", + "@article{Geiger2013IJRR,\n", + " author = {Andreas Geiger and Philip Lenz and Christoph Stiller and Raquel Urtasun},\n", + " title = {Vision meets Robotics: The KITTI Dataset},\n", + " journal = {International Journal of Robotics Research (IJRR)},\n", + " year = {2013}\n", + "}\n", + "```" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-medical-pills-dataset.ipynb b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-medical-pills-dataset.ipynb new file mode 100644 index 00000000..256fdf5b --- /dev/null +++ b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-medical-pills-dataset.ipynb @@ -0,0 +1,1701 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Train an Ultralytics YOLO detector on the medical-pills dataset and stream every sample, prediction and metric live into Weights Studio to inspect, edit and evolve your run.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Medical Pills Detection · Ultralytics YOLO + WeightsLab\n", + "\n", + "This notebook trains a YOLO detector on the [medical-pills](https://docs.ultralytics.com/) dataset (detecting medical pills for pharmaceutical quality control) **through the WeightsLab training pipeline**.\n", + "\n", + "Instead of Ultralytics' one-shot `train`/`predict` flow, we hand training to `WLAwareTrainer`. It wraps the train/val dataloaders, logs a **per-sample** signal for every image (loss, prediction stats, tags) and ships live prediction overlays to **Weights Studio** — where you inspect the data grid, remove/weight samples, edit the model and resume, all while training runs.\n", + "\n", + "Ultralytics auto-downloads the dataset the first time you train." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o7YFKEN1Ks7Y" + }, + "source": [ + "## 1 · Setup\n", + "\n", + "Install WeightsLab (which pulls in Ultralytics) and verify the environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "71be394c", + "outputId": "c9ef4767-84f5-4bea-e395-9e11766002f2", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "source": [ + "%pip install setuptools wheel" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (75.2.0)\n", + "Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (0.47.0)\n", + "Requirement already satisfied: packaging>=24.0 in /usr/local/lib/python3.12/dist-packages (from wheel) (26.2)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "%pip install git+https://github.com/GrayboxTech/weightslab.git" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "jV1oi_PQN0xK", + "outputId": "eda8eca1-f4b3-4ffd-eaa6-0f67ce857153" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting git+https://github.com/GrayboxTech/weightslab.git\n", + " Cloning https://github.com/GrayboxTech/weightslab.git (to revision landingcollab) to /tmp/pip-req-build-z4ba7cx9\n", + " Running command git clone --filter=blob:none --quiet https://github.com/GrayboxTech/weightslab.git /tmp/pip-req-build-z4ba7cx9\n", + " Running command git checkout -b landingcollab --track origin/landingcollab\n", + " Switched to a new branch 'landingcollab'\n", + " Branch 'landingcollab' set up to track remote branch 'landingcollab' from 'origin'.\n", + " Resolved https://github.com/GrayboxTech/weightslab.git to commit 3f1cf143bc93e67df43adc71ab831fa71477e0d2\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.10.2)\n", + "Requirement already satisfied: torch<=2.9,>=2.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.9.0)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.24.0)\n", + "Requirement already satisfied: torchmetrics>=1.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.21)\n", + "Requirement already satisfied: onnx<=1.20,>=1.15 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.20.0)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.4.9)\n", + "Requirement already satisfied: langchain-ollama<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.1.0)\n", + "Requirement already satisfied: langchain-openai<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.5)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev8) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.2)\n", + "Requirement already satisfied: ollama<1.0.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev8) (0.6.2)\n", + "Requirement already satisfied: openai<3.0.0,>=2.45.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2.45.0)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev8) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.5.0)\n", + "Requirement already satisfied: lightning-utilities>=0.15.3 in /usr/local/lib/python3.12/dist-packages (from torchmetrics>=1.9->weightslab==1.3.3.dev8) (0.15.3)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.0.0)\n", + "Building wheels for collected packages: weightslab\n", + " Building wheel for weightslab (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for weightslab: filename=weightslab-1.3.3.dev8-py3-none-any.whl size=2827715 sha256=5c4e7c14bbf95a48b9a8c86b5a8c744af67e1eb395197cbe79d3e443c21cf86e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-__zwgf7x/wheels/4d/c7/cd/817c2745790555e7b6c532f5b6055d47567f9416fdfc65879b\n", + "Successfully built weightslab\n", + "Installing collected packages: weightslab\n", + " Attempting uninstall: weightslab\n", + " Found existing installation: weightslab 1.3.3.dev7\n", + " Uninstalling weightslab-1.3.3.dev7:\n", + " Successfully uninstalled weightslab-1.3.3.dev7\n", + "Successfully installed weightslab-1.3.3.dev8\n" + ] + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "weightslab" + ] + }, + "id": "b7fce274042740d19087972fe9c01c9b" + } + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pJhvREbaKs7f", + "outputId": "651baa36-9ebb-49da-830b-e6c8a6e2b64f" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Ultralytics 8.4.96 🚀 Python-3.12.13 torch-2.9.0+cu128 CPU (Intel Xeon CPU @ 2.20GHz)\n", + "Setup complete ✅ (2 CPUs, 12.7 GB RAM, 30.5/107.7 GB disk)\n", + "16/07/2026-09:29:29.683 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmp6meuzqdz/weightslab_logs/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:29.685 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-09:29:29.686 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev8 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-09:29:29.690 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install weightslab\n", + "# %pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev8\"\n", + "%pip install ultralytics\n", + "\n", + "import ultralytics\n", + "ultralytics.checks()\n", + "\n", + "import weightslab as wl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 · Dataset\n", + "\n", + "The dataset is described by a small YAML file that Ultralytics resolves and downloads automatically. For reference:" + ] + }, + { + "cell_type": "markdown", + "source": [ + "```yaml\n", + "# Ultralytics YOLO 🚀, AGPL-3.0 license\n", + "# Medical-pills dataset by Ultralytics\n", + "# Documentation: https://docs.ultralytics.com/datasets/detect/medical-pills/\n", + "# Example usage: yolo train data=medical-pills.yaml\n", + "# parent\n", + "# ├── ultralytics\n", + "# └── datasets\n", + "# └── medical-pills ← downloads here (8.19 MB)\n", + "\n", + "# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..]\n", + "path: medical-pills # dataset root dir\n", + "train: images/train # train images (relative to 'path') 92 images\n", + "val: images/val # val images (relative to 'path') 23 images\n", + "\n", + "# Classes\n", + "names:\n", + " 0: pill\n", + "\n", + "# Download script/URL (optional)\n", + "download: https://github.com/ultralytics/assets/releases/download/v0.0.0/medical-pills.zip\n", + "```" + ], + "metadata": { + "id": "h8go3HNgN0WU" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rH79t30YKs7i" + }, + "source": [ + "## 3 · Train through WeightsLab\n", + "\n", + "We register the run's hyperparameters with `wl.watch_or_edit(...)` (so they become live-editable from Studio), start the backend services with `wl.serve()`, then hand training to `WLAwareTrainer` via the standard `YOLO(...).train(trainer=...)` entry point.\n", + "\n", + "> **Connect Weights Studio** — run `weightslab start` locally (opens `http://localhost:5173`) *before or during* training to watch the run live. On **Colab**, serve over a public tunnel instead: use `wl.serve(serving_grpc=True, serving_bore=True)` and connect your local Studio with `weightslab tunnel bore.pub:PORT` (the port is printed when serving starts).\n", + "\n", + "Data augmentations are disabled so each sample maps cleanly to its ground truth in the grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ.setdefault(\"WL_PRELOAD_IMAGE_OVERVIEW\", \"0\")\n", + "os.environ.setdefault(\"WEIGHTSLAB_LOG_LEVEL\", \"WARNING\")\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "from weightslab.integrations.ultralytics import WLAwareTrainer\n", + "from ultralytics import YOLO\n", + "\n", + "# Hyperparameters registered with WeightsLab -> live-editable from Weights Studio.\n", + "cfg = {\n", + " \"model\": \"yolo11n.pt\", # pretrained checkpoint\n", + " \"data\": \"medical-pills.yaml\", # Ultralytics auto-downloads\n", + " \"image_size\": 640,\n", + " \"epochs\": 20,\n", + " # Per-sample prediction overlays streamed to Studio (NMS on the train set).\n", + " \"signals_cfg\": {\n", + " \"train_nms\": {\"conf_thres\": 0.25, \"iou_thres\": 0.45, \"max_nms\": 7},\n", + " },\n", + "}\n", + "wl.watch_or_edit(cfg, flag=\"hyperparameters\", defaults=cfg, poll_interval=1.0)\n", + "\n", + "# Start the WeightsLab backend services that Weights Studio connects to.\n", + "wl.serve(serving_grpc=True)\n", + "# Colab -> local Studio: wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "wl.start_training() # equivalent to pressing \"play\" in Studio\n", + "\n", + "# Read back the (now live) hyperparameters and hand training to WLAwareTrainer.\n", + "model = YOLO(cfg[\"model\"])\n", + "results = model.train(\n", + " trainer=WLAwareTrainer,\n", + " data=str(cfg[\"data\"]),\n", + " imgsz=cfg[\"image_size\"],\n", + " epochs=int(cfg[\"epochs\"]),\n", + " project=\"./wl_logs\", name=\"medical-pills\", # -> WL log_dir/name\n", + " workers=0, # WL invariant (parent-process uid counter)\n", + " optimizer=\"SGD\", lr0=0.001, amp=False,\n", + " # All augmentations off for a clean sample <-> ground-truth association.\n", + " mosaic=0.0, mixup=0.0, copy_paste=0.0,\n", + " hsv_h=0.0, hsv_s=0.0, hsv_v=0.0,\n", + " degrees=0.0, translate=0.0, scale=0.0, shear=0.0, perspective=0.0,\n", + " flipud=0.0, fliplr=0.0, erasing=0.0, auto_augment=None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f-Uw_s_8Ks7m" + }, + "source": [ + "## 4 · Explore the data grid\n", + "\n", + "The table below is exactly the data behind Weights Studio's **grid explorer**: one row per sample, keyed by split (`origin`) and sample id, with the per-sample loss, prediction stats and tags accumulated during training. This is the ledger that Studio reads from — here we render it inline with pandas.\n", + "\n", + "The image thumbnails and bounding-box overlays themselves are rendered by **Weights Studio** (which streams full previews over gRPC); the link below opens the interactive grid there." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "collapsed": true, + "id": "uOJdXGA3Ks7o", + "outputId": "26652fa3-6a15-43ce-9d75-034eaba21ccd" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "1241 samples tracked\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " prediction prediction_raw \\\n", + "sample_id annotation_id \n", + "0 0 None None \n", + "1 0 None None \n", + "2 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "3 0 None None \n", + "4 0 None None \n", + "5 0 None None \n", + "6 0 None None \n", + "7 0 None None \n", + "8 0 None None \n", + "9 0 None None \n", + "10 0 None None \n", + "11 0 None None \n", + "12 0 None None \n", + "13 0 None None \n", + "14 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "15 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "16 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "17 0 None None \n", + "18 0 None None \n", + "19 0 None None \n", + "20 0 None None \n", + " 1 None None \n", + " 2 None None \n", + " 3 None None \n", + "21 0 None None \n", + "22 0 None None \n", + "23 0 None None \n", + "24 0 None None \n", + "25 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "26 0 None None \n", + "27 0 None None \n", + "28 0 None None \n", + "29 0 None None \n", + "30 0 None None \n", + "31 0 None None \n", + "32 0 None None \n", + "33 0 None None \n", + "34 0 None None \n", + "35 0 None None \n", + "36 0 None None \n", + "\n", + " target \\\n", + "sample_id annotation_id \n", + "0 0 [0.2335685, 0.254695, 0.4553995, 0.43075103, 1... \n", + "1 0 [0.251174, 0.24061048, 0.44366202, 0.4307515, ... \n", + "2 0 None \n", + " 1 [0.523474, 0.3978875, 0.634976, 0.48122054, 1.... \n", + " 2 [0.40610352, 0.415493, 0.5234745, 0.522301, 1.... \n", + "3 0 [0.403756, 0.3826295, 0.637324, 0.5152585, 1.0... \n", + "4 0 [0.4436615, 0.3814555, 0.5938965, 0.4518785, 1... \n", + "5 0 [0.6044605, 0.3990615, 0.67253554, 0.46596247,... \n", + "6 0 [0.410798, 0.4436625, 0.556338, 0.51173747, 1.... \n", + "7 0 [0.650235, 0.4166665, 0.73826295, 0.48356748, ... \n", + "8 0 [0.64788747, 0.388498, 0.7582165, 0.49413198, ... \n", + "9 0 [0.6807515, 0.40140802, 0.75704247, 0.45774597... \n", + "10 0 [0.431925, 0.1701875, 0.561033, 0.3180745, 1.0... \n", + "11 0 [0.42018852, 0.34507048, 0.5434275, 0.4941315,... \n", + "12 0 [0.3732395, 0.348592, 0.54812247, 0.46009403, ... \n", + "13 0 [0.37793398, 0.3814555, 0.48122, 0.45892054, 1... \n", + "14 0 None \n", + " 1 [0.43427247, 0.3556335, 0.51173747, 0.43075046... \n", + " 2 [0.4929575, 0.44718304, 0.5375585, 0.485915, 1... \n", + "15 0 None \n", + " 1 [0.629108, 0.39788702, 0.67840403, 0.453051, 1... \n", + " 2 [0.4530515, 0.4107985, 0.58568054, 0.5105635, ... \n", + "16 0 None \n", + " 1 [0.45070451, 0.3967135, 0.5762915, 0.5152585, ... \n", + " 2 [0.63732404, 0.403756, 0.684272, 0.46009403, 1... \n", + "17 0 [0.43779355, 0.3826295, 0.61032856, 0.5223005,... \n", + "18 0 [0.4401405, 0.374413, 0.6068075, 0.529343, 1.0... \n", + "19 0 [0.536385, 0.4495305, 0.600939, 0.49999946, 0.... \n", + "20 0 None \n", + " 1 [0.52347445, 0.4424885, 0.6150235, 0.5281695, ... \n", + " 2 [0.456573, 0.515258, 0.504695, 0.562206, 0.0, ... \n", + " 3 [0.504695, 0.44248852, 0.54342705, 0.48004752,... \n", + "21 0 [0.54107946, 0.456573, 0.6103285, 0.511737, 0.... \n", + "22 0 [0.3298125, 0.2042255, 0.4225355, 0.2957745, 0... \n", + "23 0 [0.517606, 0.18661949, 0.615024, 0.2875585, 1.... \n", + "24 0 [0.51173747, 0.18779299, 0.6126765, 0.308685, ... \n", + "25 0 None \n", + " 1 [0.30985948, 0.3626765, 0.3861505, 0.43896753,... \n", + " 2 [0.41314548, 0.3427225, 0.48122054, 0.42018753... \n", + "26 0 [0.25821602, 0.30046952, 0.46830997, 0.4577465... \n", + "27 0 [0.2734745, 0.2828635, 0.44718352, 0.4577465, ... \n", + "28 0 [0.5915495, 0.23591551, 0.6455405, 0.2887325, ... \n", + "29 0 [0.5903755, 0.231221, 0.6420185, 0.280517, 1.0... \n", + "30 0 [0.40493003, 0.2018775, 0.469484, 0.2699525, 1... \n", + "31 0 [0.39554, 0.19483551, 0.483568, 0.2887325, 1.0... \n", + "32 0 [0.4025825, 0.212441, 0.45539945, 0.269953, 1.... \n", + "33 0 [0.2265255, 0.2136145, 0.3438965, 0.30751148, ... \n", + "34 0 [0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0] \n", + "35 0 [0.6079815, 0.320423, 0.7969485, 0.489437, 1.0... \n", + "36 0 [0.6185445, 0.30164298, 0.7758215, 0.494131, 1... \n", + "\n", + " discarded origin task_type last_seen group_id \\\n", + "sample_id annotation_id \n", + "0 0 False train_loader -1.0 0 \n", + "1 0 False train_loader -1.0 1 \n", + "2 0 False train_loader -1.0 2 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "3 0 False train_loader -1.0 3 \n", + "4 0 False train_loader -1.0 4 \n", + "5 0 False train_loader -1.0 5 \n", + "6 0 False train_loader -1.0 6 \n", + "7 0 False train_loader -1.0 7 \n", + "8 0 False train_loader -1.0 8 \n", + "9 0 False train_loader -1.0 9 \n", + "10 0 False train_loader -1.0 10 \n", + "11 0 False train_loader -1.0 11 \n", + "12 0 False train_loader -1.0 12 \n", + "13 0 False train_loader -1.0 13 \n", + "14 0 False train_loader -1.0 14 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "15 0 False train_loader -1.0 15 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "16 0 False train_loader -1.0 16 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "17 0 False train_loader -1.0 17 \n", + "18 0 False train_loader -1.0 18 \n", + "19 0 False train_loader -1.0 19 \n", + "20 0 False train_loader -1.0 20 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + " 3 None NaN None NaN None \n", + "21 0 False train_loader -1.0 21 \n", + "22 0 False train_loader -1.0 22 \n", + "23 0 False train_loader -1.0 23 \n", + "24 0 False train_loader -1.0 24 \n", + "25 0 False train_loader -1.0 25 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "26 0 False train_loader -1.0 26 \n", + "27 0 False train_loader -1.0 27 \n", + "28 0 False train_loader -1.0 28 \n", + "29 0 False train_loader -1.0 29 \n", + "30 0 False train_loader -1.0 30 \n", + "31 0 False train_loader -1.0 31 \n", + "32 0 False train_loader -1.0 32 \n", + "33 0 False train_loader -1.0 33 \n", + "34 0 False train_loader -1.0 34 \n", + "35 0 False train_loader -1.0 35 \n", + "36 0 False train_loader -1.0 36 \n", + "\n", + " member_rank \\\n", + "sample_id annotation_id \n", + "0 0 0.0 \n", + "1 0 0.0 \n", + "2 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "3 0 0.0 \n", + "4 0 0.0 \n", + "5 0 0.0 \n", + "6 0 0.0 \n", + "7 0 0.0 \n", + "8 0 0.0 \n", + "9 0 0.0 \n", + "10 0 0.0 \n", + "11 0 0.0 \n", + "12 0 0.0 \n", + "13 0 0.0 \n", + "14 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "15 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "16 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "17 0 0.0 \n", + "18 0 0.0 \n", + "19 0 0.0 \n", + "20 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + " 3 NaN \n", + "21 0 0.0 \n", + "22 0 0.0 \n", + "23 0 0.0 \n", + "24 0 0.0 \n", + "25 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "26 0 0.0 \n", + "27 0 0.0 \n", + "28 0 0.0 \n", + "29 0 0.0 \n", + "30 0 0.0 \n", + "31 0 0.0 \n", + "32 0 0.0 \n", + "33 0 0.0 \n", + "34 0 0.0 \n", + "35 0 0.0 \n", + "36 0 0.0 \n", + "\n", + " img_path \\\n", + "sample_id annotation_id \n", + "0 0 /content/datasets/brain-tumor/images/train/000... \n", + "1 0 /content/datasets/brain-tumor/images/train/000... \n", + "2 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "3 0 /content/datasets/brain-tumor/images/train/000... \n", + "4 0 /content/datasets/brain-tumor/images/train/000... \n", + "5 0 /content/datasets/brain-tumor/images/train/000... \n", + "6 0 /content/datasets/brain-tumor/images/train/000... \n", + "7 0 /content/datasets/brain-tumor/images/train/000... \n", + "8 0 /content/datasets/brain-tumor/images/train/000... \n", + "9 0 /content/datasets/brain-tumor/images/train/000... \n", + "10 0 /content/datasets/brain-tumor/images/train/000... \n", + "11 0 /content/datasets/brain-tumor/images/train/000... \n", + "12 0 /content/datasets/brain-tumor/images/train/000... \n", + "13 0 /content/datasets/brain-tumor/images/train/000... \n", + "14 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "15 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "16 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "17 0 /content/datasets/brain-tumor/images/train/000... \n", + "18 0 /content/datasets/brain-tumor/images/train/000... \n", + "19 0 /content/datasets/brain-tumor/images/train/000... \n", + "20 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 /content/datasets/brain-tumor/images/train/000... \n", + "22 0 /content/datasets/brain-tumor/images/train/000... \n", + "23 0 /content/datasets/brain-tumor/images/train/000... \n", + "24 0 /content/datasets/brain-tumor/images/train/000... \n", + "25 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "26 0 /content/datasets/brain-tumor/images/train/000... \n", + "27 0 /content/datasets/brain-tumor/images/train/000... \n", + "28 0 /content/datasets/brain-tumor/images/train/000... \n", + "29 0 /content/datasets/brain-tumor/images/train/000... \n", + "30 0 /content/datasets/brain-tumor/images/train/000... \n", + "31 0 /content/datasets/brain-tumor/images/train/000... \n", + "32 0 /content/datasets/brain-tumor/images/train/000... \n", + "33 0 /content/datasets/brain-tumor/images/train/000... \n", + "34 0 /content/datasets/brain-tumor/images/train/000... \n", + "35 0 /content/datasets/brain-tumor/images/train/000... \n", + "36 0 /content/datasets/brain-tumor/images/train/000... \n", + "\n", + " cls \n", + "sample_id annotation_id \n", + "0 0 [[1.0]] \n", + "1 0 [[1.0]] \n", + "2 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "3 0 [[1.0]] \n", + "4 0 [[1.0]] \n", + "5 0 [[1.0]] \n", + "6 0 [[1.0]] \n", + "7 0 [[1.0]] \n", + "8 0 [[1.0]] \n", + "9 0 [[1.0]] \n", + "10 0 [[1.0]] \n", + "11 0 [[1.0]] \n", + "12 0 [[1.0]] \n", + "13 0 [[1.0]] \n", + "14 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "15 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "16 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "17 0 [[1.0]] \n", + "18 0 [[1.0]] \n", + "19 0 [[0.0]] \n", + "20 0 [[0.0], [0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 [[0.0]] \n", + "22 0 [[0.0]] \n", + "23 0 [[1.0]] \n", + "24 0 [[1.0]] \n", + "25 0 [[0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + "26 0 [[0.0]] \n", + "27 0 [[0.0]] \n", + "28 0 [[1.0]] \n", + "29 0 [[1.0]] \n", + "30 0 [[1.0]] \n", + "31 0 [[1.0]] \n", + "32 0 [[1.0]] \n", + "33 0 [[1.0]] \n", + "34 0 [[1.0]] \n", + "35 0 [[1.0]] \n", + "36 0 [[1.0]] " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
predictionprediction_rawtargetdiscardedorigintask_typelast_seengroup_idmember_rankimg_pathcls
sample_idannotation_id
00NoneNone[0.2335685, 0.254695, 0.4553995, 0.43075103, 1...Falsetrain_loader-1.000.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
10NoneNone[0.251174, 0.24061048, 0.44366202, 0.4307515, ...Falsetrain_loader-1.010.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
20NoneNoneNoneFalsetrain_loader-1.020.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.523474, 0.3978875, 0.634976, 0.48122054, 1....NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.40610352, 0.415493, 0.5234745, 0.522301, 1....NoneNaNNoneNaNNoneNaNNoneNone
30NoneNone[0.403756, 0.3826295, 0.637324, 0.5152585, 1.0...Falsetrain_loader-1.030.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
40NoneNone[0.4436615, 0.3814555, 0.5938965, 0.4518785, 1...Falsetrain_loader-1.040.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
50NoneNone[0.6044605, 0.3990615, 0.67253554, 0.46596247,...Falsetrain_loader-1.050.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
60NoneNone[0.410798, 0.4436625, 0.556338, 0.51173747, 1....Falsetrain_loader-1.060.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
70NoneNone[0.650235, 0.4166665, 0.73826295, 0.48356748, ...Falsetrain_loader-1.070.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
80NoneNone[0.64788747, 0.388498, 0.7582165, 0.49413198, ...Falsetrain_loader-1.080.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
90NoneNone[0.6807515, 0.40140802, 0.75704247, 0.45774597...Falsetrain_loader-1.090.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
100NoneNone[0.431925, 0.1701875, 0.561033, 0.3180745, 1.0...Falsetrain_loader-1.0100.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
110NoneNone[0.42018852, 0.34507048, 0.5434275, 0.4941315,...Falsetrain_loader-1.0110.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
120NoneNone[0.3732395, 0.348592, 0.54812247, 0.46009403, ...Falsetrain_loader-1.0120.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
130NoneNone[0.37793398, 0.3814555, 0.48122, 0.45892054, 1...Falsetrain_loader-1.0130.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
140NoneNoneNoneFalsetrain_loader-1.0140.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.43427247, 0.3556335, 0.51173747, 0.43075046...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4929575, 0.44718304, 0.5375585, 0.485915, 1...NoneNaNNoneNaNNoneNaNNoneNone
150NoneNoneNoneFalsetrain_loader-1.0150.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.629108, 0.39788702, 0.67840403, 0.453051, 1...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4530515, 0.4107985, 0.58568054, 0.5105635, ...NoneNaNNoneNaNNoneNaNNoneNone
160NoneNoneNoneFalsetrain_loader-1.0160.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.45070451, 0.3967135, 0.5762915, 0.5152585, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.63732404, 0.403756, 0.684272, 0.46009403, 1...NoneNaNNoneNaNNoneNaNNoneNone
170NoneNone[0.43779355, 0.3826295, 0.61032856, 0.5223005,...Falsetrain_loader-1.0170.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
180NoneNone[0.4401405, 0.374413, 0.6068075, 0.529343, 1.0...Falsetrain_loader-1.0180.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
190NoneNone[0.536385, 0.4495305, 0.600939, 0.49999946, 0....Falsetrain_loader-1.0190.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
200NoneNoneNoneFalsetrain_loader-1.0200.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0], [0.0]]
1NoneNone[0.52347445, 0.4424885, 0.6150235, 0.5281695, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.456573, 0.515258, 0.504695, 0.562206, 0.0, ...NoneNaNNoneNaNNoneNaNNoneNone
3NoneNone[0.504695, 0.44248852, 0.54342705, 0.48004752,...NoneNaNNoneNaNNoneNaNNoneNone
210NoneNone[0.54107946, 0.456573, 0.6103285, 0.511737, 0....Falsetrain_loader-1.0210.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
220NoneNone[0.3298125, 0.2042255, 0.4225355, 0.2957745, 0...Falsetrain_loader-1.0220.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
230NoneNone[0.517606, 0.18661949, 0.615024, 0.2875585, 1....Falsetrain_loader-1.0230.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
240NoneNone[0.51173747, 0.18779299, 0.6126765, 0.308685, ...Falsetrain_loader-1.0240.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
250NoneNoneNoneFalsetrain_loader-1.0250.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0]]
1NoneNone[0.30985948, 0.3626765, 0.3861505, 0.43896753,...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.41314548, 0.3427225, 0.48122054, 0.42018753...NoneNaNNoneNaNNoneNaNNoneNone
260NoneNone[0.25821602, 0.30046952, 0.46830997, 0.4577465...Falsetrain_loader-1.0260.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
270NoneNone[0.2734745, 0.2828635, 0.44718352, 0.4577465, ...Falsetrain_loader-1.0270.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
280NoneNone[0.5915495, 0.23591551, 0.6455405, 0.2887325, ...Falsetrain_loader-1.0280.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
290NoneNone[0.5903755, 0.231221, 0.6420185, 0.280517, 1.0...Falsetrain_loader-1.0290.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
300NoneNone[0.40493003, 0.2018775, 0.469484, 0.2699525, 1...Falsetrain_loader-1.0300.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
310NoneNone[0.39554, 0.19483551, 0.483568, 0.2887325, 1.0...Falsetrain_loader-1.0310.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
320NoneNone[0.4025825, 0.212441, 0.45539945, 0.269953, 1....Falsetrain_loader-1.0320.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
330NoneNone[0.2265255, 0.2136145, 0.3438965, 0.30751148, ...Falsetrain_loader-1.0330.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
340NoneNone[0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0]Falsetrain_loader-1.0340.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
350NoneNone[0.6079815, 0.320423, 0.7969485, 0.489437, 1.0...Falsetrain_loader-1.0350.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
360NoneNone[0.6185445, 0.30164298, 0.7758215, 0.494131, 1...Falsetrain_loader-1.0360.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "repr_error": "Out of range float values are not JSON compliant: nan" + } + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "🔗 Open the full grid in Weights Studio" + ] + }, + "metadata": {} + } + ], + "source": [ + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "from weightslab.backend.ledgers import get_dataframe\n", + "\n", + "# `get_df_view()` is the same per-sample DataFrame that powers Studio's grid.\n", + "dfm = get_dataframe()\n", + "grid = dfm.get_df_view() if hasattr(dfm, \"get_df_view\") else pd.DataFrame()\n", + "print(f\"{len(grid)} samples tracked\")\n", + "display(grid.head(50))\n", + "\n", + "# Open the live, interactive grid (with image + bbox previews) in Weights Studio.\n", + "# Studio must be running locally (`weightslab start`).\n", + "display(HTML(\n", + " '🔗 Open the full grid in Weights Studio'\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FXQ0gBOSKs7t" + }, + "source": [ + "> **Can clicking a grid row here jump to that sample in Studio?** Not out of the box today. The notebook and Studio are separate processes, and the link above opens Studio at its root. Per-sample deep-linking (e.g. `http://localhost:5173/?sample=` selecting and scrolling to that row) is a small frontend addition to Weights Studio — the sample `uid` is already the grid's index here, so the notebook side is ready for it once Studio accepts the query param." + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Citation\n", + "\n", + "```bibtex\n", + "@dataset{Jocher_Ultralytics_Datasets_2024,\n", + " author = {Jocher, Glenn and Rizwan, Muhammad},\n", + " license = {AGPL-3.0},\n", + " month = {Dec},\n", + " title = {Ultralytics Datasets:Medical-pills Detection Dataset},\n", + " url = {https://docs.ultralytics.com/datasets/detect/medical-pills/},\n", + " version = {1.0.0},\n", + " year = {2024}\n", + "}\n" + ], + "metadata": { + "id": "vlHp09Nueb3d" + } + } + ] +} diff --git a/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-package-segmentation-dataset.ipynb b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-package-segmentation-dataset.ipynb new file mode 100644 index 00000000..60431dea --- /dev/null +++ b/weightslab/examples/Notebooks/Ultralytics/wl-how-to-train-ultralytics-yolo-on-package-segmentation-dataset.ipynb @@ -0,0 +1,1683 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Train an Ultralytics YOLO segmentation model on the package-seg dataset and stream every sample, prediction and metric live into Weights Studio to inspect, edit and evolve your run.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Package Segmentation · Ultralytics YOLO + WeightsLab\n", + "\n", + "This notebook trains a YOLO segmentation model on the [package-seg](https://docs.ultralytics.com/) dataset (segmenting shipping packages on conveyors for logistics automation) **through the WeightsLab training pipeline**.\n", + "\n", + "Instead of Ultralytics' one-shot `train`/`predict` flow, we hand training to `WLAwareSegmentationTrainer`. It wraps the train/val dataloaders, logs a **per-sample** signal for every image (loss, prediction stats, tags) and ships live prediction overlays to **Weights Studio** — where you inspect the data grid, remove/weight samples, edit the model and resume, all while training runs.\n", + "\n", + "Ultralytics auto-downloads the dataset the first time you train." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o7YFKEN1Ks7Y" + }, + "source": [ + "## 1 · Setup\n", + "\n", + "Install WeightsLab (which pulls in Ultralytics) and verify the environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "71be394c", + "outputId": "c9ef4767-84f5-4bea-e395-9e11766002f2", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "source": [ + "%pip install setuptools wheel" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (75.2.0)\n", + "Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (0.47.0)\n", + "Requirement already satisfied: packaging>=24.0 in /usr/local/lib/python3.12/dist-packages (from wheel) (26.2)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "%pip install git+https://github.com/GrayboxTech/weightslab.git" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "jV1oi_PQN0xK", + "outputId": "eda8eca1-f4b3-4ffd-eaa6-0f67ce857153" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting git+https://github.com/GrayboxTech/weightslab.git\n", + " Cloning https://github.com/GrayboxTech/weightslab.git (to revision landingcollab) to /tmp/pip-req-build-z4ba7cx9\n", + " Running command git clone --filter=blob:none --quiet https://github.com/GrayboxTech/weightslab.git /tmp/pip-req-build-z4ba7cx9\n", + " Running command git checkout -b landingcollab --track origin/landingcollab\n", + " Switched to a new branch 'landingcollab'\n", + " Branch 'landingcollab' set up to track remote branch 'landingcollab' from 'origin'.\n", + " Resolved https://github.com/GrayboxTech/weightslab.git to commit 3f1cf143bc93e67df43adc71ab831fa71477e0d2\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: numpy<3,>=1.24 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.0.2)\n", + "Requirement already satisfied: pandas<3,>=2.2.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.2.2)\n", + "Requirement already satisfied: duckdb<2,>=1.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.2)\n", + "Requirement already satisfied: PyYAML<7,>=6.0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (6.0.3)\n", + "Requirement already satisfied: dill<0.5,>=0.3.8 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.3.8)\n", + "Requirement already satisfied: zstandard<1,>=0.22 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.25.0)\n", + "Requirement already satisfied: h5py<4,>=3.10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.16.0)\n", + "Requirement already satisfied: xxhash<4.1,>=3.4 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.7.1)\n", + "Requirement already satisfied: tables<4,>=3.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (3.10.2)\n", + "Requirement already satisfied: torch<=2.9,>=2.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.9.0)\n", + "Requirement already satisfied: torchvision<1,>=0.16 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.24.0)\n", + "Requirement already satisfied: torchmetrics>=1.9 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: grpcio<2,>=1.80 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.81.1)\n", + "Requirement already satisfied: protobuf<8,>=5.28.1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (5.29.6)\n", + "Requirement already satisfied: pydantic<3,>=2.7 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (2.13.4)\n", + "Requirement already satisfied: Pillow<12,>=10 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (11.3.0)\n", + "Requirement already satisfied: graphviz<1,>=0.20 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (0.21)\n", + "Requirement already satisfied: onnx<=1.20,>=1.15 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.20.0)\n", + "Requirement already satisfied: tqdm<5,>=4.66 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (4.67.3)\n", + "Requirement already satisfied: python-dotenv<2,>=1 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.2.2)\n", + "Requirement already satisfied: langchain-core<2,>=0.3 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.4.9)\n", + "Requirement already satisfied: langchain-ollama<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.1.0)\n", + "Requirement already satisfied: langchain-openai<2,>=0.2 in /usr/local/lib/python3.12/dist-packages (from weightslab==1.3.3.dev8) (1.3.5)\n", + "Requirement already satisfied: typing-extensions~=4.12 in /usr/local/lib/python3.12/dist-packages (from grpcio<2,>=1.80->weightslab==1.3.3.dev8) (4.15.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.17 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.0.18)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.9.1)\n", + "Requirement already satisfied: packaging>=23.2.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (26.2)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (9.1.4)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.2)\n", + "Requirement already satisfied: ollama<1.0.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from langchain-ollama<2,>=0.2->weightslab==1.3.3.dev8) (0.6.2)\n", + "Requirement already satisfied: openai<3.0.0,>=2.45.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2.45.0)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /usr/local/lib/python3.12/dist-packages (from langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.13.0)\n", + "Requirement already satisfied: ml_dtypes>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from onnx<=1.20,>=1.15->weightslab==1.3.3.dev8) (0.5.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (2026.2)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3,>=2.7->weightslab==1.3.3.dev8) (0.4.2)\n", + "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (2.14.1)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (9.0.0)\n", + "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/lib/python3.12/dist-packages (from tables<4,>=3.9->weightslab==1.3.3.dev8) (4.5.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.29.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.1.6)\n", + "Requirement already satisfied: fsspec>=0.8.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2025.3.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (9.10.2.21)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.4.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.3.3.83)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (10.3.9.90)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (11.7.3.90)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.5.8.93)\n", + "Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (0.7.1)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (2.27.5)\n", + "Requirement already satisfied: nvidia-nvshmem-cu12==3.3.20 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.3.20)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.90)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (12.8.93)\n", + "Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.13.1.3)\n", + "Requirement already satisfied: triton==3.5.0 in /usr/local/lib/python3.12/dist-packages (from torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.5.0)\n", + "Requirement already satisfied: lightning-utilities>=0.15.3 in /usr/local/lib/python3.12/dist-packages (from torchmetrics>=1.9->weightslab==1.3.3.dev8) (0.15.3)\n", + "Requirement already satisfied: ndindex in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.10.1)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.2.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.32.4)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (13.9.4)\n", + "Requirement already satisfied: textual in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (6.2.1)\n", + "Requirement already satisfied: textual-plotext in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (1.0.1)\n", + "Requirement already satisfied: threadpoolctl in /usr/local/lib/python3.12/dist-packages (from blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.6.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.12/dist-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.1.1)\n", + "Requirement already satisfied: anyio>=3.5.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (4.14.0)\n", + "Requirement already satisfied: distro>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.28.1)\n", + "Requirement already satisfied: orjson>=3.9.14 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.11.9)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.3.1)\n", + "Requirement already satisfied: websockets>=15.0 in /usr/local/lib/python3.12/dist-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (15.0.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /usr/local/lib/python3.12/dist-packages (from openai<3.0.0,>=2.45.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (0.15.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas<3,>=2.2.2->weightslab==1.3.3.dev8) (1.17.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (1.3.0)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.12/dist-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2,>=0.2->weightslab==1.3.3.dev8) (2025.11.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch<=2.9,>=2.1->weightslab==1.3.3.dev8) (3.0.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.12/dist-packages (from anyio>=3.5.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (3.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<1.0.0,>=0.3.45->langchain-core<2,>=0.3->weightslab==1.3.3.dev8) (0.16.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (3.4.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.5.0)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.20.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.6.0 in /usr/local/lib/python3.12/dist-packages (from textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (4.10.0)\n", + "Requirement already satisfied: plotext<6.0.0,>=5.2.8 in /usr/local/lib/python3.12/dist-packages (from textual-plotext->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (5.3.2)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.1.0)\n", + "Requirement already satisfied: mdit-py-plugins>=0.5.0 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (0.6.1)\n", + "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.12/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual->blosc2>=2.3.0->tables<4,>=3.9->weightslab==1.3.3.dev8) (2.0.0)\n", + "Building wheels for collected packages: weightslab\n", + " Building wheel for weightslab (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for weightslab: filename=weightslab-1.3.3.dev8-py3-none-any.whl size=2827715 sha256=5c4e7c14bbf95a48b9a8c86b5a8c744af67e1eb395197cbe79d3e443c21cf86e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-__zwgf7x/wheels/4d/c7/cd/817c2745790555e7b6c532f5b6055d47567f9416fdfc65879b\n", + "Successfully built weightslab\n", + "Installing collected packages: weightslab\n", + " Attempting uninstall: weightslab\n", + " Found existing installation: weightslab 1.3.3.dev7\n", + " Uninstalling weightslab-1.3.3.dev7:\n", + " Successfully uninstalled weightslab-1.3.3.dev7\n", + "Successfully installed weightslab-1.3.3.dev8\n" + ] + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "weightslab" + ] + }, + "id": "b7fce274042740d19087972fe9c01c9b" + } + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pJhvREbaKs7f", + "outputId": "651baa36-9ebb-49da-830b-e6c8a6e2b64f" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Ultralytics 8.4.96 🚀 Python-3.12.13 torch-2.9.0+cu128 CPU (Intel Xeon CPU @ 2.20GHz)\n", + "Setup complete ✅ (2 CPUs, 12.7 GB RAM, 30.5/107.7 GB disk)\n", + "16/07/2026-09:29:29.683 INFO:root:setup_logging: WeightsLab logging initialized - Log file: /tmp/tmp6meuzqdz/weightslab_logs/weightslab_20260716_092929.log\n", + "16/07/2026-09:29:29.685 INFO:weightslab:: WeightsLab package initialized - Log level: INFO, Log to file: True\n", + "16/07/2026-09:29:29.686 INFO:weightslab:: \n", + "╭─────────────────────────────────────────────────────────────────────────────────────────────────────╮\n", + "│ │\n", + "│ \u001b[32m$$\\ $$\\ \u001b[0m $$\\ $$\\ $$\\ \u001b[31m$$\\ \u001b[0m $$\\ │\n", + "│ \u001b[32m$$ | $\\ $$ |\u001b[0m \\__| $$ | $$ | \u001b[31m$$ | \u001b[0m $$ | │\n", + "│ \u001b[32m$$ |$$$\\ $$ |\u001b[0m $$$$$$\\ $$\\ $$$$$$\\ $$$$$$$\\ $$$$$$\\ $$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$\\ $$$$$$$\\ │\n", + "│ \u001b[32m$$ $$ $$\\$$ |\u001b[0m$$ __$$\\ $$ |$$ __$$\\ $$ __$$\\ \\_$$ _| $$ _____|\u001b[31m$$ | \u001b[0m \\____$$\\ $$ __$$\\ │\n", + "│ \u001b[32m$$$$ _$$$$ |\u001b[0m$$$$$$$$ |$$ |$$ / $$ |$$ | $$ | $$ | \\$$$$$$\\ \u001b[31m$$ | \u001b[0m $$$$$$$ |$$ | $$ | │\n", + "│ \u001b[32m$$$ / \\$$$ |\u001b[0m$$ ____|$$ |$$ | $$ |$$ | $$ | $$ |$$\\ \\____$$\\ \u001b[31m$$ | \u001b[0m$$ __$$ |$$ | $$ | │\n", + "│ \u001b[32m$$ / \\$$ |\u001b[0m\\$$$$$$$\\ $$ |\\$$$$$$$ |$$ | $$ | \\$$$$ |$$$$$$$ |\u001b[31m$$$$$$$$\\ \u001b[0m\\$$$$$$$ |$$$$$$$ | │\n", + "│ \u001b[32m\\__/ \\__|\u001b[0m \\_______|\\__| \\____$$ |\\__| \\__| \\____/ \\_______/ \u001b[31m\\________|\u001b[0m \\_______|\\_______/ │\n", + "│ \u001b[32m \u001b[0m $$\\ $$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\$$$$$$ |\u001b[31m\u001b[0m │\n", + "│ \u001b[32m \u001b[0m \\______/\u001b[31m\u001b[0m │\n", + "│ │\n", + "│ Inspect - Edit - Evolve Neural Networks │\n", + "│ │\n", + "╰─── By GrayBx, v1.3.3.dev8 ──────────────────────────────────────────────────────────────────────────╯\n", + "\n", + "\n", + "16/07/2026-09:29:29.690 INFO:weightslab.utils.telemetry:ping_import: WeightsLab uses anonymous usage data (package version used, and OS name). Set WL_NO_TELEMETRY=1 to disable.\n" + ] + } + ], + "source": [ + "# Install WeightsLab. Colab already ships torch, torchvision, numpy, scikit-learn\n", + "# and Pillow, so nothing extra is needed here.\n", + "# %pip install weightslab\n", + "# %pip install --pre --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ \"weightslab==1.3.3.dev8\"\n", + "%pip install ultralytics\n", + "\n", + "import ultralytics\n", + "ultralytics.checks()\n", + "\n", + "import weightslab as wl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 · Dataset\n", + "\n", + "The dataset is described by a small YAML file that Ultralytics resolves and downloads automatically. For reference:" + ] + }, + { + "cell_type": "markdown", + "source": [ + "```yaml\n", + "# Ultralytics YOLO 🚀, AGPL-3.0 license\n", + "# Package-seg dataset by Ultralytics\n", + "# Documentation: https://docs.ultralytics.com/datasets/segment/package-seg/\n", + "# Example usage: yolo train data=package-seg.yaml\n", + "# parent\n", + "# ├── ultralytics\n", + "# └── datasets\n", + "# └── package-seg ← downloads here (103 MB)\n", + "\n", + "# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..]\n", + "path: package-seg # dataset root dir\n", + "train: images/train # train images (relative to 'path') 1920 images\n", + "val: images/val # val images (relative to 'path') 89 images\n", + "test: images/test # test images (relative to 'path') 188 images\n", + "\n", + "# Classes\n", + "names:\n", + " 0: package\n", + "\n", + "# Download script/URL (optional)\n", + "download: https://github.com/ultralytics/assets/releases/download/v0.0.0/package-seg.zip\n", + "```" + ], + "metadata": { + "id": "h8go3HNgN0WU" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rH79t30YKs7i" + }, + "source": [ + "## 3 · Train through WeightsLab\n", + "\n", + "We register the run's hyperparameters with `wl.watch_or_edit(...)` (so they become live-editable from Studio), start the backend services with `wl.serve()`, then hand training to `WLAwareTrainer` via the standard `YOLO(...).train(trainer=...)` entry point.\n", + "\n", + "> **Connect Weights Studio** — run `weightslab start` locally (opens `http://localhost:5173`) *before or during* training to watch the run live. On **Colab**, serve over a public tunnel instead: use `wl.serve(serving_grpc=True, serving_bore=True)` and connect your local Studio with `weightslab tunnel bore.pub:PORT` (the port is printed when serving starts).\n", + "\n", + "Data augmentations are disabled so each sample maps cleanly to its ground truth in the grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ.setdefault(\"WL_PRELOAD_IMAGE_OVERVIEW\", \"0\")\n", + "os.environ.setdefault(\"WEIGHTSLAB_LOG_LEVEL\", \"WARNING\")\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "from weightslab.integrations.ultralytics import WLAwareSegmentationTrainer\n", + "from ultralytics import YOLO\n", + "\n", + "# Hyperparameters registered with WeightsLab -> live-editable from Weights Studio.\n", + "cfg = {\n", + " \"model\": \"yolo11n-seg.pt\", # pretrained checkpoint\n", + " \"data\": \"package-seg.yaml\", # Ultralytics auto-downloads\n", + " \"image_size\": 640,\n", + " \"epochs\": 10,\n", + " # Per-sample prediction overlays streamed to Studio (NMS on the train set).\n", + " \"signals_cfg\": {\n", + " \"train_nms\": {\"conf_thres\": 0.25, \"iou_thres\": 0.45, \"max_nms\": 7},\n", + " },\n", + "}\n", + "wl.watch_or_edit(cfg, flag=\"hyperparameters\", defaults=cfg, poll_interval=1.0)\n", + "\n", + "# Start the WeightsLab backend services that Weights Studio connects to.\n", + "wl.serve(serving_grpc=True)\n", + "# Colab -> local Studio: wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "wl.start_training() # equivalent to pressing \"play\" in Studio\n", + "\n", + "# Read back the (now live) hyperparameters and hand training to WLAwareSegmentationTrainer.\n", + "model = YOLO(cfg[\"model\"])\n", + "results = model.train(\n", + " trainer=WLAwareSegmentationTrainer,\n", + " data=str(cfg[\"data\"]),\n", + " imgsz=cfg[\"image_size\"],\n", + " epochs=int(cfg[\"epochs\"]),\n", + " project=\"./wl_logs\", name=\"package-seg\", # -> WL log_dir/name\n", + " workers=0, # WL invariant (parent-process uid counter)\n", + " optimizer=\"SGD\", lr0=0.001, amp=False,\n", + " # All augmentations off for a clean sample <-> ground-truth association.\n", + " mosaic=0.0, mixup=0.0, copy_paste=0.0,\n", + " hsv_h=0.0, hsv_s=0.0, hsv_v=0.0,\n", + " degrees=0.0, translate=0.0, scale=0.0, shear=0.0, perspective=0.0,\n", + " flipud=0.0, fliplr=0.0, erasing=0.0, auto_augment=None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f-Uw_s_8Ks7m" + }, + "source": [ + "## 4 · Explore the data grid\n", + "\n", + "The table below is exactly the data behind Weights Studio's **grid explorer**: one row per sample, keyed by split (`origin`) and sample id, with the per-sample loss, prediction stats and tags accumulated during training. This is the ledger that Studio reads from — here we render it inline with pandas.\n", + "\n", + "The image thumbnails and bounding-box overlays themselves are rendered by **Weights Studio** (which streams full previews over gRPC); the link below opens the interactive grid there." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "collapsed": true, + "id": "uOJdXGA3Ks7o", + "outputId": "26652fa3-6a15-43ce-9d75-034eaba21ccd" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "1241 samples tracked\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " prediction prediction_raw \\\n", + "sample_id annotation_id \n", + "0 0 None None \n", + "1 0 None None \n", + "2 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "3 0 None None \n", + "4 0 None None \n", + "5 0 None None \n", + "6 0 None None \n", + "7 0 None None \n", + "8 0 None None \n", + "9 0 None None \n", + "10 0 None None \n", + "11 0 None None \n", + "12 0 None None \n", + "13 0 None None \n", + "14 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "15 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "16 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "17 0 None None \n", + "18 0 None None \n", + "19 0 None None \n", + "20 0 None None \n", + " 1 None None \n", + " 2 None None \n", + " 3 None None \n", + "21 0 None None \n", + "22 0 None None \n", + "23 0 None None \n", + "24 0 None None \n", + "25 0 None None \n", + " 1 None None \n", + " 2 None None \n", + "26 0 None None \n", + "27 0 None None \n", + "28 0 None None \n", + "29 0 None None \n", + "30 0 None None \n", + "31 0 None None \n", + "32 0 None None \n", + "33 0 None None \n", + "34 0 None None \n", + "35 0 None None \n", + "36 0 None None \n", + "\n", + " target \\\n", + "sample_id annotation_id \n", + "0 0 [0.2335685, 0.254695, 0.4553995, 0.43075103, 1... \n", + "1 0 [0.251174, 0.24061048, 0.44366202, 0.4307515, ... \n", + "2 0 None \n", + " 1 [0.523474, 0.3978875, 0.634976, 0.48122054, 1.... \n", + " 2 [0.40610352, 0.415493, 0.5234745, 0.522301, 1.... \n", + "3 0 [0.403756, 0.3826295, 0.637324, 0.5152585, 1.0... \n", + "4 0 [0.4436615, 0.3814555, 0.5938965, 0.4518785, 1... \n", + "5 0 [0.6044605, 0.3990615, 0.67253554, 0.46596247,... \n", + "6 0 [0.410798, 0.4436625, 0.556338, 0.51173747, 1.... \n", + "7 0 [0.650235, 0.4166665, 0.73826295, 0.48356748, ... \n", + "8 0 [0.64788747, 0.388498, 0.7582165, 0.49413198, ... \n", + "9 0 [0.6807515, 0.40140802, 0.75704247, 0.45774597... \n", + "10 0 [0.431925, 0.1701875, 0.561033, 0.3180745, 1.0... \n", + "11 0 [0.42018852, 0.34507048, 0.5434275, 0.4941315,... \n", + "12 0 [0.3732395, 0.348592, 0.54812247, 0.46009403, ... \n", + "13 0 [0.37793398, 0.3814555, 0.48122, 0.45892054, 1... \n", + "14 0 None \n", + " 1 [0.43427247, 0.3556335, 0.51173747, 0.43075046... \n", + " 2 [0.4929575, 0.44718304, 0.5375585, 0.485915, 1... \n", + "15 0 None \n", + " 1 [0.629108, 0.39788702, 0.67840403, 0.453051, 1... \n", + " 2 [0.4530515, 0.4107985, 0.58568054, 0.5105635, ... \n", + "16 0 None \n", + " 1 [0.45070451, 0.3967135, 0.5762915, 0.5152585, ... \n", + " 2 [0.63732404, 0.403756, 0.684272, 0.46009403, 1... \n", + "17 0 [0.43779355, 0.3826295, 0.61032856, 0.5223005,... \n", + "18 0 [0.4401405, 0.374413, 0.6068075, 0.529343, 1.0... \n", + "19 0 [0.536385, 0.4495305, 0.600939, 0.49999946, 0.... \n", + "20 0 None \n", + " 1 [0.52347445, 0.4424885, 0.6150235, 0.5281695, ... \n", + " 2 [0.456573, 0.515258, 0.504695, 0.562206, 0.0, ... \n", + " 3 [0.504695, 0.44248852, 0.54342705, 0.48004752,... \n", + "21 0 [0.54107946, 0.456573, 0.6103285, 0.511737, 0.... \n", + "22 0 [0.3298125, 0.2042255, 0.4225355, 0.2957745, 0... \n", + "23 0 [0.517606, 0.18661949, 0.615024, 0.2875585, 1.... \n", + "24 0 [0.51173747, 0.18779299, 0.6126765, 0.308685, ... \n", + "25 0 None \n", + " 1 [0.30985948, 0.3626765, 0.3861505, 0.43896753,... \n", + " 2 [0.41314548, 0.3427225, 0.48122054, 0.42018753... \n", + "26 0 [0.25821602, 0.30046952, 0.46830997, 0.4577465... \n", + "27 0 [0.2734745, 0.2828635, 0.44718352, 0.4577465, ... \n", + "28 0 [0.5915495, 0.23591551, 0.6455405, 0.2887325, ... \n", + "29 0 [0.5903755, 0.231221, 0.6420185, 0.280517, 1.0... \n", + "30 0 [0.40493003, 0.2018775, 0.469484, 0.2699525, 1... \n", + "31 0 [0.39554, 0.19483551, 0.483568, 0.2887325, 1.0... \n", + "32 0 [0.4025825, 0.212441, 0.45539945, 0.269953, 1.... \n", + "33 0 [0.2265255, 0.2136145, 0.3438965, 0.30751148, ... \n", + "34 0 [0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0] \n", + "35 0 [0.6079815, 0.320423, 0.7969485, 0.489437, 1.0... \n", + "36 0 [0.6185445, 0.30164298, 0.7758215, 0.494131, 1... \n", + "\n", + " discarded origin task_type last_seen group_id \\\n", + "sample_id annotation_id \n", + "0 0 False train_loader -1.0 0 \n", + "1 0 False train_loader -1.0 1 \n", + "2 0 False train_loader -1.0 2 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "3 0 False train_loader -1.0 3 \n", + "4 0 False train_loader -1.0 4 \n", + "5 0 False train_loader -1.0 5 \n", + "6 0 False train_loader -1.0 6 \n", + "7 0 False train_loader -1.0 7 \n", + "8 0 False train_loader -1.0 8 \n", + "9 0 False train_loader -1.0 9 \n", + "10 0 False train_loader -1.0 10 \n", + "11 0 False train_loader -1.0 11 \n", + "12 0 False train_loader -1.0 12 \n", + "13 0 False train_loader -1.0 13 \n", + "14 0 False train_loader -1.0 14 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "15 0 False train_loader -1.0 15 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "16 0 False train_loader -1.0 16 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "17 0 False train_loader -1.0 17 \n", + "18 0 False train_loader -1.0 18 \n", + "19 0 False train_loader -1.0 19 \n", + "20 0 False train_loader -1.0 20 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + " 3 None NaN None NaN None \n", + "21 0 False train_loader -1.0 21 \n", + "22 0 False train_loader -1.0 22 \n", + "23 0 False train_loader -1.0 23 \n", + "24 0 False train_loader -1.0 24 \n", + "25 0 False train_loader -1.0 25 \n", + " 1 None NaN None NaN None \n", + " 2 None NaN None NaN None \n", + "26 0 False train_loader -1.0 26 \n", + "27 0 False train_loader -1.0 27 \n", + "28 0 False train_loader -1.0 28 \n", + "29 0 False train_loader -1.0 29 \n", + "30 0 False train_loader -1.0 30 \n", + "31 0 False train_loader -1.0 31 \n", + "32 0 False train_loader -1.0 32 \n", + "33 0 False train_loader -1.0 33 \n", + "34 0 False train_loader -1.0 34 \n", + "35 0 False train_loader -1.0 35 \n", + "36 0 False train_loader -1.0 36 \n", + "\n", + " member_rank \\\n", + "sample_id annotation_id \n", + "0 0 0.0 \n", + "1 0 0.0 \n", + "2 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "3 0 0.0 \n", + "4 0 0.0 \n", + "5 0 0.0 \n", + "6 0 0.0 \n", + "7 0 0.0 \n", + "8 0 0.0 \n", + "9 0 0.0 \n", + "10 0 0.0 \n", + "11 0 0.0 \n", + "12 0 0.0 \n", + "13 0 0.0 \n", + "14 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "15 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "16 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "17 0 0.0 \n", + "18 0 0.0 \n", + "19 0 0.0 \n", + "20 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + " 3 NaN \n", + "21 0 0.0 \n", + "22 0 0.0 \n", + "23 0 0.0 \n", + "24 0 0.0 \n", + "25 0 0.0 \n", + " 1 NaN \n", + " 2 NaN \n", + "26 0 0.0 \n", + "27 0 0.0 \n", + "28 0 0.0 \n", + "29 0 0.0 \n", + "30 0 0.0 \n", + "31 0 0.0 \n", + "32 0 0.0 \n", + "33 0 0.0 \n", + "34 0 0.0 \n", + "35 0 0.0 \n", + "36 0 0.0 \n", + "\n", + " img_path \\\n", + "sample_id annotation_id \n", + "0 0 /content/datasets/brain-tumor/images/train/000... \n", + "1 0 /content/datasets/brain-tumor/images/train/000... \n", + "2 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "3 0 /content/datasets/brain-tumor/images/train/000... \n", + "4 0 /content/datasets/brain-tumor/images/train/000... \n", + "5 0 /content/datasets/brain-tumor/images/train/000... \n", + "6 0 /content/datasets/brain-tumor/images/train/000... \n", + "7 0 /content/datasets/brain-tumor/images/train/000... \n", + "8 0 /content/datasets/brain-tumor/images/train/000... \n", + "9 0 /content/datasets/brain-tumor/images/train/000... \n", + "10 0 /content/datasets/brain-tumor/images/train/000... \n", + "11 0 /content/datasets/brain-tumor/images/train/000... \n", + "12 0 /content/datasets/brain-tumor/images/train/000... \n", + "13 0 /content/datasets/brain-tumor/images/train/000... \n", + "14 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "15 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "16 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "17 0 /content/datasets/brain-tumor/images/train/000... \n", + "18 0 /content/datasets/brain-tumor/images/train/000... \n", + "19 0 /content/datasets/brain-tumor/images/train/000... \n", + "20 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 /content/datasets/brain-tumor/images/train/000... \n", + "22 0 /content/datasets/brain-tumor/images/train/000... \n", + "23 0 /content/datasets/brain-tumor/images/train/000... \n", + "24 0 /content/datasets/brain-tumor/images/train/000... \n", + "25 0 /content/datasets/brain-tumor/images/train/000... \n", + " 1 None \n", + " 2 None \n", + "26 0 /content/datasets/brain-tumor/images/train/000... \n", + "27 0 /content/datasets/brain-tumor/images/train/000... \n", + "28 0 /content/datasets/brain-tumor/images/train/000... \n", + "29 0 /content/datasets/brain-tumor/images/train/000... \n", + "30 0 /content/datasets/brain-tumor/images/train/000... \n", + "31 0 /content/datasets/brain-tumor/images/train/000... \n", + "32 0 /content/datasets/brain-tumor/images/train/000... \n", + "33 0 /content/datasets/brain-tumor/images/train/000... \n", + "34 0 /content/datasets/brain-tumor/images/train/000... \n", + "35 0 /content/datasets/brain-tumor/images/train/000... \n", + "36 0 /content/datasets/brain-tumor/images/train/000... \n", + "\n", + " cls \n", + "sample_id annotation_id \n", + "0 0 [[1.0]] \n", + "1 0 [[1.0]] \n", + "2 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "3 0 [[1.0]] \n", + "4 0 [[1.0]] \n", + "5 0 [[1.0]] \n", + "6 0 [[1.0]] \n", + "7 0 [[1.0]] \n", + "8 0 [[1.0]] \n", + "9 0 [[1.0]] \n", + "10 0 [[1.0]] \n", + "11 0 [[1.0]] \n", + "12 0 [[1.0]] \n", + "13 0 [[1.0]] \n", + "14 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "15 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "16 0 [[1.0], [1.0]] \n", + " 1 None \n", + " 2 None \n", + "17 0 [[1.0]] \n", + "18 0 [[1.0]] \n", + "19 0 [[0.0]] \n", + "20 0 [[0.0], [0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + " 3 None \n", + "21 0 [[0.0]] \n", + "22 0 [[0.0]] \n", + "23 0 [[1.0]] \n", + "24 0 [[1.0]] \n", + "25 0 [[0.0], [0.0]] \n", + " 1 None \n", + " 2 None \n", + "26 0 [[0.0]] \n", + "27 0 [[0.0]] \n", + "28 0 [[1.0]] \n", + "29 0 [[1.0]] \n", + "30 0 [[1.0]] \n", + "31 0 [[1.0]] \n", + "32 0 [[1.0]] \n", + "33 0 [[1.0]] \n", + "34 0 [[1.0]] \n", + "35 0 [[1.0]] \n", + "36 0 [[1.0]] " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
predictionprediction_rawtargetdiscardedorigintask_typelast_seengroup_idmember_rankimg_pathcls
sample_idannotation_id
00NoneNone[0.2335685, 0.254695, 0.4553995, 0.43075103, 1...Falsetrain_loader-1.000.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
10NoneNone[0.251174, 0.24061048, 0.44366202, 0.4307515, ...Falsetrain_loader-1.010.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
20NoneNoneNoneFalsetrain_loader-1.020.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.523474, 0.3978875, 0.634976, 0.48122054, 1....NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.40610352, 0.415493, 0.5234745, 0.522301, 1....NoneNaNNoneNaNNoneNaNNoneNone
30NoneNone[0.403756, 0.3826295, 0.637324, 0.5152585, 1.0...Falsetrain_loader-1.030.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
40NoneNone[0.4436615, 0.3814555, 0.5938965, 0.4518785, 1...Falsetrain_loader-1.040.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
50NoneNone[0.6044605, 0.3990615, 0.67253554, 0.46596247,...Falsetrain_loader-1.050.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
60NoneNone[0.410798, 0.4436625, 0.556338, 0.51173747, 1....Falsetrain_loader-1.060.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
70NoneNone[0.650235, 0.4166665, 0.73826295, 0.48356748, ...Falsetrain_loader-1.070.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
80NoneNone[0.64788747, 0.388498, 0.7582165, 0.49413198, ...Falsetrain_loader-1.080.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
90NoneNone[0.6807515, 0.40140802, 0.75704247, 0.45774597...Falsetrain_loader-1.090.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
100NoneNone[0.431925, 0.1701875, 0.561033, 0.3180745, 1.0...Falsetrain_loader-1.0100.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
110NoneNone[0.42018852, 0.34507048, 0.5434275, 0.4941315,...Falsetrain_loader-1.0110.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
120NoneNone[0.3732395, 0.348592, 0.54812247, 0.46009403, ...Falsetrain_loader-1.0120.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
130NoneNone[0.37793398, 0.3814555, 0.48122, 0.45892054, 1...Falsetrain_loader-1.0130.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
140NoneNoneNoneFalsetrain_loader-1.0140.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.43427247, 0.3556335, 0.51173747, 0.43075046...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4929575, 0.44718304, 0.5375585, 0.485915, 1...NoneNaNNoneNaNNoneNaNNoneNone
150NoneNoneNoneFalsetrain_loader-1.0150.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.629108, 0.39788702, 0.67840403, 0.453051, 1...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.4530515, 0.4107985, 0.58568054, 0.5105635, ...NoneNaNNoneNaNNoneNaNNoneNone
160NoneNoneNoneFalsetrain_loader-1.0160.0/content/datasets/brain-tumor/images/train/000...[[1.0], [1.0]]
1NoneNone[0.45070451, 0.3967135, 0.5762915, 0.5152585, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.63732404, 0.403756, 0.684272, 0.46009403, 1...NoneNaNNoneNaNNoneNaNNoneNone
170NoneNone[0.43779355, 0.3826295, 0.61032856, 0.5223005,...Falsetrain_loader-1.0170.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
180NoneNone[0.4401405, 0.374413, 0.6068075, 0.529343, 1.0...Falsetrain_loader-1.0180.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
190NoneNone[0.536385, 0.4495305, 0.600939, 0.49999946, 0....Falsetrain_loader-1.0190.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
200NoneNoneNoneFalsetrain_loader-1.0200.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0], [0.0]]
1NoneNone[0.52347445, 0.4424885, 0.6150235, 0.5281695, ...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.456573, 0.515258, 0.504695, 0.562206, 0.0, ...NoneNaNNoneNaNNoneNaNNoneNone
3NoneNone[0.504695, 0.44248852, 0.54342705, 0.48004752,...NoneNaNNoneNaNNoneNaNNoneNone
210NoneNone[0.54107946, 0.456573, 0.6103285, 0.511737, 0....Falsetrain_loader-1.0210.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
220NoneNone[0.3298125, 0.2042255, 0.4225355, 0.2957745, 0...Falsetrain_loader-1.0220.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
230NoneNone[0.517606, 0.18661949, 0.615024, 0.2875585, 1....Falsetrain_loader-1.0230.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
240NoneNone[0.51173747, 0.18779299, 0.6126765, 0.308685, ...Falsetrain_loader-1.0240.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
250NoneNoneNoneFalsetrain_loader-1.0250.0/content/datasets/brain-tumor/images/train/000...[[0.0], [0.0]]
1NoneNone[0.30985948, 0.3626765, 0.3861505, 0.43896753,...NoneNaNNoneNaNNoneNaNNoneNone
2NoneNone[0.41314548, 0.3427225, 0.48122054, 0.42018753...NoneNaNNoneNaNNoneNaNNoneNone
260NoneNone[0.25821602, 0.30046952, 0.46830997, 0.4577465...Falsetrain_loader-1.0260.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
270NoneNone[0.2734745, 0.2828635, 0.44718352, 0.4577465, ...Falsetrain_loader-1.0270.0/content/datasets/brain-tumor/images/train/000...[[0.0]]
280NoneNone[0.5915495, 0.23591551, 0.6455405, 0.2887325, ...Falsetrain_loader-1.0280.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
290NoneNone[0.5903755, 0.231221, 0.6420185, 0.280517, 1.0...Falsetrain_loader-1.0290.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
300NoneNone[0.40493003, 0.2018775, 0.469484, 0.2699525, 1...Falsetrain_loader-1.0300.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
310NoneNone[0.39554, 0.19483551, 0.483568, 0.2887325, 1.0...Falsetrain_loader-1.0310.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
320NoneNone[0.4025825, 0.212441, 0.45539945, 0.269953, 1....Falsetrain_loader-1.0320.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
330NoneNone[0.2265255, 0.2136145, 0.3438965, 0.30751148, ...Falsetrain_loader-1.0330.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
340NoneNone[0.634976, 0.349765, 0.7723, 0.456573, 1.0, 1.0]Falsetrain_loader-1.0340.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
350NoneNone[0.6079815, 0.320423, 0.7969485, 0.489437, 1.0...Falsetrain_loader-1.0350.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
360NoneNone[0.6185445, 0.30164298, 0.7758215, 0.494131, 1...Falsetrain_loader-1.0360.0/content/datasets/brain-tumor/images/train/000...[[1.0]]
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "repr_error": "Out of range float values are not JSON compliant: nan" + } + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "🔗 Open the full grid in Weights Studio" + ] + }, + "metadata": {} + } + ], + "source": [ + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "from weightslab.backend.ledgers import get_dataframe\n", + "\n", + "# `get_df_view()` is the same per-sample DataFrame that powers Studio's grid.\n", + "dfm = get_dataframe()\n", + "grid = dfm.get_df_view() if hasattr(dfm, \"get_df_view\") else pd.DataFrame()\n", + "print(f\"{len(grid)} samples tracked\")\n", + "display(grid.head(50))\n", + "\n", + "# Open the live, interactive grid (with image + bbox previews) in Weights Studio.\n", + "# Studio must be running locally (`weightslab start`).\n", + "display(HTML(\n", + " '🔗 Open the full grid in Weights Studio'\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FXQ0gBOSKs7t" + }, + "source": [ + "> **Can clicking a grid row here jump to that sample in Studio?** Not out of the box today. The notebook and Studio are separate processes, and the link above opens Studio at its root. Per-sample deep-linking (e.g. `http://localhost:5173/?sample=` selecting and scrolling to that row) is a small frontend addition to Weights Studio — the sample `uid` is already the grid's index here, so the notebook side is ready for it once Studio accepts the query param." + ] + } + ] +} diff --git a/weightslab/examples/Notebooks/Usecases/wl-segmentation-loss-shapes-classification.ipynb b/weightslab/examples/Notebooks/Usecases/wl-segmentation-loss-shapes-classification.ipynb new file mode 100644 index 00000000..c2d2a723 --- /dev/null +++ b/weightslab/examples/Notebooks/Usecases/wl-segmentation-loss-shapes-classification.ipynb @@ -0,0 +1,530 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + " \n", + " \"WeightsLab\n", + "\n", + " \"License\"\n", + " \"Stars\"\n", + " \"Version\"\n", + "
\n", + " \"Open\n", + "\n", + " Welcome to the WeightsLab custom decorated-signal usecase! This notebook trains segmentation and adds a user-defined, decorated per-sample signal that classifies each sample's loss trajectory. See the Docs for details.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Segmentation + a custom decorated per-sample signal\n", + "\n", + "This usecase trains the **BDD100k segmentation** example *in the notebook kernel* and layers on a\n", + "**user-defined signal** decorated with `@wl.signal`. The signal **subscribes** to the per-sample\n", + "segmentation loss, classifies each sample's loss **trajectory shape** (monotonic, plateaued,\n", + "spiked, ...), and writes the verdict back as a categorical tag - all live, per sample.\n", + "\n", + "It also demonstrates the `min_step` parameter of `@wl.signal`: the classifier only starts firing\n", + "once each sample has enough history (here `min_step=505`).\n", + "\n", + "> Why in-kernel (not `!python main.py`)? A decorated `@wl.signal` must be registered in the **same\n", + "> process** that runs training, so we build and train here rather than shelling out." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Pick a **T4 GPU** runtime on Colab. We clone the repo (for the segmentation `utils/` and the\n", + "bundled `BDD_subset/` data) and install WeightsLab from the clone (needed for the `min_step`\n", + "parameter)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"PyPI\n", + "\"PyPI\n", + "\"PyPI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 1 · Install WeightsLab (+ Colab compatibility overrides)\n", + "# Colab already has a compatible torch/torchvision + numpy 1.26 — do NOT reinstall them.\n", + "# One resolution, pin numpy<2 and protobuf into weightslab's range.\n", + "%pip install weightslab # --extra-index-url https://download.pytorch.org/whl/cpu\n", + "\n", + "# 2 · Import\n", + "import weightslab as wl\n", + "\n", + "# 3 · Serve to Weights Studio over a public bore tunnel.\n", + "# Prints a `bore.pub:PORT` endpoint. Then, on your own machine (Docker running):\n", + "# weightslab start # opens http://localhost:5173\n", + "# weightslab tunnel bore.pub:PORT # PORT = the port printed below\n", + "wl.serve(serving_grpc=True, serving_bore=True)\n", + "\n", + "# 4 · Register first the configuration then YOUR training objects here (uncomment and fill in):\n", + "# wl.watch_or_edit(config, flag=\"hyperparameters\", poll_interval=1.0)\n", + "# wl.watch_or_edit(model, optimizer, train_loader, test_loader, ...)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Imports\n", + "\n", + "We reuse the segmentation example's helper modules (`utils/`) for the dataset, model, and the\n", + "per-sample / per-instance Dice + BCE criterions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "from torch import optim\n", + "from tqdm.auto import tqdm\n", + "\n", + "from weightslab.components.global_monitoring import (\n", + " guard_training_context, guard_testing_context,\n", + ")\n", + "from utils.data import BDD100kSegDataset, seg_collate\n", + "from utils.model import SmallUNet\n", + "from utils.criterions import (\n", + " PerSampleDice, PerInstanceDice, PerSampleBCE, PerInstanceBCE,\n", + ")\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "print(\"Using device:\", device)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Configuration\n", + "\n", + "All tunables in one dict (like a `config.yaml`, with comments)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = {\n", + " # -- Experiment ----------------------------------------------------------\n", + " \"experiment_name\": \"seg_loss_shapes\", # name shown in Weights Studio\n", + " \"root_log_dir\": None, # None -> a temp dir is created\n", + "\n", + " # -- Data (bundled BDD_subset) -------------------------------------------\n", + " \"data_root\": \"BDD_subset\", # ships in the repo\n", + " \"num_classes\": 6, # BDD label set\n", + " \"class_names\": [\"Background\", \"Ego Road\", \"Driveable Area\",\n", + " \"Lane Line 1\", \"Lane Line 2\", \"Lane Line 3\"],\n", + " \"ignore_index\": 255, # void pixels\n", + " \"image_size\": 128, # images resized to image_size^2\n", + " \"max_train_samples\": 256, # cap for a quick demo\n", + " \"max_test_samples\": 100,\n", + " \"train_batch_size\": 2,\n", + " \"test_batch_size\": 2,\n", + "\n", + " # -- Optimizer -----------------------------------------------------------\n", + " \"learning_rate\": 1e-3,\n", + "\n", + " # -- Training schedule ---------------------------------------------------\n", + " \"training_steps_to_do\": 800, # > min_step so the classifier fires\n", + " \"eval_full_to_train_steps_ratio\": 100, # full eval every N steps\n", + " \"write_export_ratio\": 100, # export history + dataframe every N steps\n", + "\n", + " # -- Custom decorated signal --------------------------------------------\n", + " \"shape_subscribe_to\": \"train_bce/sample\", # per-sample loss the classifier watches\n", + " \"shape_compute_every_n_steps\": 25, # throttle the classifier\n", + " \"shape_min_step\": 505, # WARM-UP: don't classify before this step\n", + "\n", + " # -- Services / live UI --------------------------------------------------\n", + " \"serving_grpc\": True,\n", + " \"expose_ui\": True,\n", + " \"backend_port\": 50051,\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Build data, model, optimizer, and the tracked seg signals\n", + "\n", + "Same wrapping pattern as the segmentation example: dataloaders, model, optimizer, and the\n", + "per-sample / per-instance Dice (metric) + BCE (loss) signals are all passed through\n", + "`wl.watch_or_edit(...)`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_ds = BDD100kSegDataset(root=config[\"data_root\"], split=\"train\",\n", + " num_classes=config[\"num_classes\"], class_names=config[\"class_names\"],\n", + " ignore_index=config[\"ignore_index\"], image_size=config[\"image_size\"],\n", + " max_samples=config[\"max_train_samples\"])\n", + "val_ds = BDD100kSegDataset(root=config[\"data_root\"], split=\"val\",\n", + " num_classes=config[\"num_classes\"], class_names=config[\"class_names\"],\n", + " ignore_index=config[\"ignore_index\"], image_size=config[\"image_size\"],\n", + " max_samples=config[\"max_test_samples\"])\n", + "\n", + "wl.watch_or_edit(config, flag=\"hyperparameters\", name=config[\"experiment_name\"],\n", + " defaults=config, poll_interval=1.0)\n", + "\n", + "train_loader = wl.watch_or_edit(\n", + " train_ds, flag=\"data\", loader_name=\"train_loader\",\n", + " batch_size=config[\"train_batch_size\"], shuffle=True, is_training=True,\n", + " compute_hash=False, array_autoload_arrays=False, array_return_proxies=True,\n", + " array_use_cache=True, preload_labels=False, collate_fn=seg_collate)\n", + "test_loader = wl.watch_or_edit(\n", + " val_ds, flag=\"data\", loader_name=\"test_loader\",\n", + " batch_size=config[\"test_batch_size\"], shuffle=False, is_training=False,\n", + " compute_hash=False, array_autoload_arrays=False, array_return_proxies=True,\n", + " array_use_cache=True, preload_labels=True, collate_fn=seg_collate)\n", + "\n", + "model = wl.watch_or_edit(\n", + " SmallUNet(in_channels=3, num_classes=config[\"num_classes\"],\n", + " image_size=config[\"image_size\"]).to(device),\n", + " flag=\"model\", device=device, compute_dependencies=True)\n", + "optimizer = wl.watch_or_edit(\n", + " optim.Adam(model.parameters(), lr=config[\"learning_rate\"]), flag=\"optimizer\")\n", + "\n", + "\n", + "def make_seg_signals(split):\n", + " return {\n", + " \"dice_sample\": wl.watch_or_edit(PerSampleDice(), flag=\"metric\",\n", + " name=f\"{split}_dice/sample\", per_sample=True, log=True),\n", + " \"dice_instance\": wl.watch_or_edit(PerInstanceDice(), flag=\"metric\",\n", + " name=f\"{split}_dice/instance\", per_instance=True, log=True),\n", + " \"bce_sample\": wl.watch_or_edit(PerSampleBCE(), flag=\"loss\",\n", + " name=f\"{split}_bce/sample\", per_sample=True, log=True),\n", + " \"bce_instance\": wl.watch_or_edit(PerInstanceBCE(), flag=\"loss\",\n", + " name=f\"{split}_bce/instance\", per_instance=True, log=True),\n", + " }\n", + "\n", + "train_sig = make_seg_signals(\"train\")\n", + "test_sig = make_seg_signals(\"test\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. The custom, user-defined decorated signal\n", + "\n", + "This is the point of the usecase. `classify_loss_shape` is **your** function, decorated with\n", + "`@wl.signal`. Because it sets `subscribe_to=`, WeightsLab calls it **once per\n", + "sample** each time that loss is logged. It pulls the sample's full loss history, classifies the\n", + "**shape** of the curve, and tags the sample.\n", + "\n", + "Note `min_step=505`: the classifier is **skipped until training step 505**, so it only runs once\n", + "each sample has a meaningful trajectory. `compute_every_n_steps=25` then throttles it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Allowed categorical values (registered so the UI shows all choices up front).\n", + "LOSS_SHAPE_LABELS = [\"monotonic\", \"plateaued\", \"Flat_high\",\n", + " \"high_variance\", \"U_Shape\", \"Spiked\"]\n", + "LOSS_SHAPE_CODES = {label: i for i, label in enumerate(LOSS_SHAPE_LABELS)}\n", + "wl.register_categorical_tag(\"loss_shape\", LOSS_SHAPE_LABELS)\n", + "\n", + "_MIN_HISTORY = 5\n", + "\n", + "def _classify_loss_shape(values):\n", + " \"\"\"Classify a per-sample loss trajectory (scale-invariant thresholds).\"\"\"\n", + " y = np.asarray(values, dtype=float)\n", + " if y.size < _MIN_HISTORY:\n", + " return None\n", + " n = y.size\n", + " first, last = float(y[0]), float(y[-1])\n", + " ymin, ymax = float(y.min()), float(y.max())\n", + " rng = max(ymax - ymin, 1e-8)\n", + " mean = float(y.mean())\n", + " cv = float(y.std()) / (abs(mean) + 1e-8)\n", + " drop = (first - last) / (abs(first) + 1e-8)\n", + " argmin = int(np.argmin(y))\n", + " rebound = (last - ymin) / rng\n", + " max_up_jump = float(np.diff(y).max()) / rng\n", + " tail = y[int(0.6 * n):]\n", + " tail_flat = (float(tail.std()) / (abs(float(tail.mean())) + 1e-8)) < 0.1\n", + " if max_up_jump > 0.5:\n", + " return \"Spiked\"\n", + " if cv > 0.5:\n", + " return \"high_variance\"\n", + " if 0.2 * n < argmin < 0.8 * n and rebound > 0.3:\n", + " return \"U_Shape\"\n", + " if drop > 0.4:\n", + " return \"monotonic\"\n", + " if drop > 0.15 and tail_flat:\n", + " return \"plateaued\"\n", + " return \"Flat_high\"\n", + "\n", + "\n", + "@wl.signal(\n", + " name=\"seg_loss_shape_classifier\",\n", + " subscribe_to=config[\"shape_subscribe_to\"],\n", + " compute_every_n_steps=config[\"shape_compute_every_n_steps\"],\n", + " min_step=config[\"shape_min_step\"], # <-- warm-up gate: skip until this step\n", + " log=False, # side-effecting: we tag samples, no aggregate curve\n", + ")\n", + "def classify_loss_shape(ctx: wl.SignalContext) -> int:\n", + " \"\"\"Per-sample: classify the loss curve's shape and tag the sample.\"\"\"\n", + " history = wl.query_sample_history(\n", + " ctx.sample_id, signal_name=config[\"shape_subscribe_to\"],\n", + " exp_hash=wl.get_current_experiment_hash())\n", + " series = sorted(((step, val) for _, step, val, _ in history), key=lambda t: t[0])\n", + " values = [v for _, v in series]\n", + " label = _classify_loss_shape(values)\n", + " if label is None:\n", + " return -1\n", + " wl.set_categorical_tag([ctx.sample_id], \"loss_shape\", label)\n", + " return LOSS_SHAPE_CODES[label]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Train / eval steps" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def _run_signals(sig, outputs, labels, ids, preds):\n", + " bce_sample = sig[\"bce_sample\"](outputs, labels, batch_ids=ids, preds=preds)\n", + " dice_sample = sig[\"dice_sample\"](outputs, labels, batch_ids=ids)\n", + " sig[\"dice_instance\"](outputs, labels, batch_ids=ids)\n", + " sig[\"bce_instance\"](outputs, labels, batch_ids=ids)\n", + " avg = 0.5 * dice_sample + 0.5 * bce_sample\n", + " wl.save_signals({\"combined_bce_dice_per_sample\": avg}, ids)\n", + " return avg, dice_sample\n", + "\n", + "\n", + "def train_step(loader, model, optimizer, sig):\n", + " with guard_training_context:\n", + " inputs, ids, labels, _ = next(loader)\n", + " inputs = inputs.to(device)\n", + " labels = [[m.to(device) for m in insts] for insts in labels]\n", + " optimizer.zero_grad()\n", + " outputs = model(inputs)\n", + " preds = outputs.argmax(dim=1)\n", + " loss_per_sample, _ = _run_signals(sig, outputs, labels, ids, preds)\n", + " loss = loss_per_sample.mean()\n", + " loss.backward()\n", + " optimizer.step()\n", + " return float(loss.detach().cpu().item())\n", + "\n", + "\n", + "def evaluate(loader, model, sig, n_batches):\n", + " losses = dices = 0.0\n", + " with guard_testing_context, torch.no_grad():\n", + " for inputs, ids, labels, _ in loader:\n", + " inputs = inputs.to(device)\n", + " labels = [[m.to(device) for m in insts] for insts in labels]\n", + " outputs = model(inputs)\n", + " preds = outputs.argmax(dim=1)\n", + " loss_per_sample, dice_sample = _run_signals(sig, outputs, labels, ids, preds)\n", + " losses += torch.mean(loss_per_sample)\n", + " dices += torch.mean(dice_sample)\n", + " return float((losses / n_batches)), float((dices / n_batches) * 100.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. (Optional) Expose the backend for the live UI\n", + "\n", + "Run this to open a raw-TCP [`bore`](https://github.com/ekzhang/bore) tunnel (`bore.pub`, no signup)\n", + "so a Weights Studio on your machine can connect. In Studio you'll see samples getting tagged with\n", + "their `loss_shape` **after step 505**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os, re, tarfile, threading, urllib.request, subprocess\n", + "\n", + "endpoint = None\n", + "if config[\"expose_ui\"]:\n", + " bore = os.path.join(os.getcwd(), \"bore\")\n", + " if not os.path.exists(bore):\n", + " BORE = \"v0.6.0\"\n", + " urllib.request.urlretrieve(\n", + " f\"https://github.com/ekzhang/bore/releases/download/{BORE}/bore-{BORE}-x86_64-unknown-linux-musl.tar.gz\",\n", + " \"bore.tar.gz\")\n", + " with tarfile.open(\"bore.tar.gz\") as t:\n", + " t.extractall()\n", + " os.chmod(bore, 0o755)\n", + " proc = subprocess.Popen([bore, \"local\", str(config[\"backend_port\"]), \"--to\", \"bore.pub\"],\n", + " stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)\n", + " for line in proc.stdout:\n", + " m = re.search(r\"bore\\.pub:(\\d+)\", line)\n", + " if m:\n", + " endpoint = f\"bore.pub:{m.group(1)}\"\n", + " break\n", + " threading.Thread(target=lambda: [None for _ in proc.stdout], daemon=True).start()\n", + " print(\"=\" * 60)\n", + " print(\"Backend exposed at:\", endpoint)\n", + " print(\"On your machine: weightslab start && weightslab tunnel\", endpoint)\n", + " print(\"=\" * 60)\n", + "else:\n", + " print(\"expose_ui is False - no tunnel.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Train\n", + "\n", + "Watch the progress bar: before step 505 the `seg_loss_shape_classifier` stays silent; once the\n", + "warm-up gate opens it starts tagging samples by loss shape (visible in Studio and in the exported\n", + "dataframe)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wl.start_training(timeout=3)\n", + "\n", + "steps = config[\"training_steps_to_do\"]\n", + "eval_ratio = config[\"eval_full_to_train_steps_ratio\"]\n", + "export_ratio = config[\"write_export_ratio\"]\n", + "\n", + "test_loss = test_dice = None\n", + "pbar = tqdm(range(steps), desc=\"Training\")\n", + "for step in pbar:\n", + " age = model.get_age() if hasattr(model, \"get_age\") else step\n", + " tr_loss = train_step(train_loader, model, optimizer, train_sig)\n", + " if age > 0 and age % eval_ratio == 0:\n", + " test_loss, test_dice = evaluate(test_loader, model, test_sig, len(test_loader))\n", + " if age > 0 and age % export_ratio == 0:\n", + " wl.write_history(); wl.write_dataframe()\n", + " post = {\"loss\": f\"{tr_loss:.3f}\", \"shapes\": \"on\" if age >= config[\"shape_min_step\"] else \"warmup\"}\n", + " if test_dice is not None:\n", + " post[\"dice\"] = f\"{test_dice:.1f}%\"\n", + " pbar.set_postfix(post)\n", + "\n", + "wl.write_history(); wl.write_dataframe()\n", + "print(\"Done.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Inspect the loss-shape tags\n", + "\n", + "After step 505 the classifier tags each sample. Export shows the `tag:loss_shape` column and the\n", + "numeric `seg_loss_shape_classifier` signal - group or sort by them in Studio to triage samples by\n", + "how their loss behaved." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import glob, pandas as pd\n", + "lg = wl.write_dataframe() # returns the written path\n", + "paths = sorted(glob.glob(os.path.join(os.path.dirname(lg), \"*dataframe*.json\")), key=os.path.getmtime)\n", + "if paths:\n", + " df = pd.read_json(paths[-1])\n", + " shape_cols = [c for c in df.columns if \"loss_shape\" in c.lower()]\n", + " print(\"shape-related columns:\", shape_cols)\n", + " display(df[shape_cols].head(30) if shape_cols else df.head())\n", + "else:\n", + " print(\"No export found yet.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## See it live in Weights Studio\n", + "\n", + "**Colab has no Docker daemon**, so run Studio on your own machine and point it at this backend with\n", + "the `bore.pub:` printed in Section 6:\n", + "\n", + "```bash\n", + "pip install weightslab\n", + "weightslab ui launch # Terminal 1 -> http://localhost:5173\n", + "weightslab tunnel bore.pub:12345 # Terminal 2 -> host:port from Section 6\n", + "```\n", + "\n", + "Group the sample grid by the `loss_shape` tag to see which samples plateaued, spiked, or never\n", + "learned - the decorated signal wrote those verdicts live, starting at step 505." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "
\n", + "Crafted by GrayboxTech - if WeightsLab helps you catch a bad label, drop us a star on GitHub.\n", + "
" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "wl-segmentation-loss-shapes-classification.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/weightslab/examples/PyTorch/wl-ads-recommendation/README.md b/weightslab/examples/PyTorch/wl-ads-recommendation/README.md new file mode 100644 index 00000000..2e8d9032 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-ads-recommendation/README.md @@ -0,0 +1,102 @@ +# WeightsLab — Advertising CTR Recommendation (tabular, pure PyTorch) + +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GrayboxTech/weightslab/blob/main/weightslab/examples/Notebooks/PyTorch/wl-ads-recommendation.ipynb) + +A fully-runnable **click-through-rate (CTR) prediction** example — the core of +an advertising recommendation system. A **Wide & Deep** model learns to predict +`P(click)` for `(user, ad, context)` impressions, streaming per-impression loss / +prediction / accuracy to the WeightsLab UI. Being tabular, it's a natural fit +for the **List Exploration (tabular) view** — sort by `loss` or `prediction` to +inspect the impressions the model ranks best and worst. + +Everything is plain PyTorch + NumPy — no external download. The impressions are +generated in-process so runs are reproducible and offline. + +## Quick start + +```bash +cd weightslab/examples/PyTorch/wl-ads-recommendation +pip install -r requirements.txt +python main.py +``` + +Then open the UI (e.g. `http://localhost:5173`), switch the Data Exploration +board to **List** view, press play, and sort columns to explore. + +## The data & model + +`make_synthetic_ctr(n, seed)` (in `utils/data.py`) generates reproducible ad +impressions with: + +* **8 categorical fields** — `user_segment`, `ad_category`, `device_type`, `os`, + `publisher`, `placement`, `region`, `hour_bucket` (embedded by the model). +* **8 numeric features** — `ad_position`, `bid_price`, `user_age`, + `session_depth`, `historical_ctr`, … (standardized). +* a binary `clicked` label at a calibrated **~20% CTR**, driven by per-field + effects plus an `ad_category × user_segment` interaction. + +The 16 field values are packed into a `1x4x4` tensor per impression (heatmap +thumbnail in the grid); the model unpacks them back into indices + numerics. + +**Model — Wide & Deep** (`utils/model.py`, after Cheng et al., 2016): +* *Deep*: per-field embeddings ⊕ numeric → MLP (generalizes feature interactions). +* *Wide*: first-order per-category effects + linear numeric term (memorizes + strong direct signals). +The two logit heads are summed. Related architectures: DeepFM, Factorization +Machines. + +### Using a real dataset + +Swap `AdsCTRDataset` for a loader over a real CTR log and set +`CATEGORICAL_CARDINALITIES` to your vocab sizes: + +* **Criteo Display Advertising** — 13 numeric + 26 categorical fields. + +* **Avazu CTR** — +* **MovieLens** (recommender variant) — + +Keep the `__getitem__` contract `(packed_features_as_image, idx, label)`. + +## What "a sample" is here + +There are no images — **each sample is one ad impression (a row)**, and the +model input **is** the packed 1-D field vector (not a reshaped image). +WeightsLab carries that vector through gRPC as a `raw_data` stat of type +`vector` (the actual values), so `inputs`, `labels`/`target` and `metadata` all +reach the UI. The 8 categorical fields (as readable labels) and 8 numeric +features are also exposed as **sortable columns** via the dataset's +`get_items()` metadata contract (`preload_metadata=True`), so the List +Exploration view shows real tabular columns (`ad_category`, `placement`, +`bid_price`, …) alongside the tracked stats below. Sort, lock, histograms, +discard/restore and neuron ops all work as on MNIST — they operate on the +per-sample ledger, not on pixels. + +## What you'll see in the UI + +| Signal / column | Meaning | +| ---------------------------------------- | --------------------------------------------- | +| field columns (`ad_category`, `placement`, `bid_price`, …) | The 16 impression fields, sortable | +| `train-loss-CE`, `test-loss-CE` | Weighted cross-entropy per split | +| `metric-ACC` | Overall accuracy | +| `test_metric/PredictedCTR_per_sample` | Model's predicted `P(click)` per impression | +| `test_metric/Accuracy_per_sample` | Per-impression correctness (0/1) | +| `target`, `prediction` columns | Per-sample truth/pred to sort/lock in List view | + +## Test it + +```bash +# Fast, offline unit tests (pure PyTorch, no gRPC server): +python -m pytest test_ads_recommendation.py -v + +# End-to-end integration check (needs weightslab installed): drives the tracked +# loaders + watched loss + gRPC server, then asserts the ledger dataframe the UI +# reads has per-sample rows, every field as a column, target/prediction/loss, +# and a live gRPC endpoint. +python verify_integration.py +``` + +The unit tests cover schema, reproducibility, calibrated CTR, pack/unpack +roundtrip, `get_items` metadata columns, the model forward pass, and that +training reduces loss and **ranks real clicks above non-clicks** on a held-out +split (rank-AUC > 0.6). `verify_integration.py` proves WeightsLab is fully wired +in tabular mode. diff --git a/weightslab/examples/PyTorch/wl-ads-recommendation/config.yaml b/weightslab/examples/PyTorch/wl-ads-recommendation/config.yaml new file mode 100644 index 00000000..856e6f1c --- /dev/null +++ b/weightslab/examples/PyTorch/wl-ads-recommendation/config.yaml @@ -0,0 +1,52 @@ +# Global configuration +experiment_name: ads_ctr_wide_deep +device: auto +training_steps_to_do: null # null = train until manually stopped from the UI +# root_log_dir: ... + +compute_natural_sort: false + +# Experiment parameters +eval_full_to_train_steps_ratio: 100 +experiment_dump_to_train_steps_ratio: 250 +write_export_ratio: 100 +skip_checkpoint_load: false +tqdm_display: true +is_training: false # Start paused; press play in the UI. + +# Global dataframe storage +ledger_enable_flushing_threads: true +ledger_enable_h5_persistence: true +ledger_flush_max_rows: 15000 +ledger_flush_interval: 30.0 + +# Clients +serving_grpc: true + +# Model (Wide & Deep) +model: + emb_dim: 8 + hidden: 128 + +# Optimizer +optimizer: + lr: 0.005 + +# Class weighting to counter ~20% click prevalence [no-click, click]. +class_weights: [1.0, 4.0] + +# Synthetic dataset generation (reproducible, no download). +dataset: + seed: 0 + n_train: 8000 + n_test: 2000 + +# DataLoader parameters +data: + train_loader: + shuffle: true + batch_size: 64 + test_loader: + shuffle: false + batch_size: 256 + drop_last: false diff --git a/weightslab/examples/PyTorch/wl-ads-recommendation/main.py b/weightslab/examples/PyTorch/wl-ads-recommendation/main.py new file mode 100644 index 00000000..4b864c41 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-ads-recommendation/main.py @@ -0,0 +1,284 @@ +"""WeightsLab example: advertising CTR recommendation (tabular, PyTorch). + +A Wide & Deep model predicts click-through-rate over synthetic ad impressions +(8 categorical fields + 8 numeric features), wired into WeightsLab so the run +streams per-impression stats (loss, prediction, target) to the UI. Being +tabular, it pairs naturally with the List Exploration (tabular) view — sort by +loss or prediction to inspect the impressions the model ranks best/worst. + +Run: + cd weightslab/examples/PyTorch/wl-ads-recommendation + python main.py + +The dataset/model live in ``utils/`` (pure PyTorch) so they can be unit tested +without the gRPC backend — see ``test_ads_recommendation.py``. +""" + +import itertools +import os +import time +import logging +import tempfile + +import yaml +import tqdm +import torch +import torch.nn as nn +import torch.optim as optim + +from torchmetrics.classification import Accuracy + +import weightslab as wl +from weightslab.components.global_monitoring import ( + guard_training_context, + guard_testing_context, +) + +from utils.data import AdsCTRDataset, CATEGORICAL_CARDINALITIES, NUM_NUMERIC +from utils.model import WideDeepCTR + + +logging.basicConfig(level=logging.ERROR) +logger = logging.getLogger(__name__) + + +# ----------------------------------------------------------------------------- +# Train / Test steps +# ----------------------------------------------------------------------------- +def train(loader, model, optimizer, criterion_mlt, device): + """Single training step using the tracked dataloader + watched loss.""" + with guard_training_context: + (inputs, ids, labels) = next(loader) + inputs = inputs.to(device) + labels = labels.to(device) + + optimizer.zero_grad() + preds_raw = model(inputs) + preds = preds_raw.argmax(dim=1, keepdim=True) + + loss_batch_mlt = criterion_mlt( + preds_raw.float(), + labels.long(), + batch_ids=ids, + preds=preds, + ) + total_loss = loss_batch_mlt.mean() + + total_loss.backward() + optimizer.step() + + return total_loss.detach().cpu().item() + + +def test(loader, model, criterion_mlt, metric_mlt, device, test_loader_len): + """Full evaluation pass over the test loader, logging per-sample signals.""" + losses = torch.tensor(0.0, device=device) + + for (inputs, ids, labels) in loader: + with guard_testing_context: + inputs = inputs.to(device) + labels = labels.to(device) + + outputs = model(inputs) + preds = outputs.argmax(dim=1, keepdim=True) + + loss_batch = criterion_mlt(outputs, labels, batch_ids=ids, preds=preds) + losses += torch.mean(loss_batch) + metric_mlt.update(outputs, labels) + + preds_flat = preds.view(-1) + labels_flat = labels.view(-1) + prob_click = torch.softmax(outputs, dim=1)[:, 1] + correct_per_sample = (preds_flat == labels_flat).float() + + signals = { + "test_metric/Accuracy_per_sample": correct_per_sample, + "test_metric/PredictedCTR_per_sample": prob_click, + } + wl.save_signals( + preds_raw=outputs, + targets=labels, + batch_ids=ids, + signals=signals, + preds=preds, + ) + + loss = losses / max(1, test_loader_len) + metric = metric_mlt.compute() * 100 + + return loss.detach().cpu().item(), metric.detach().cpu().item() + + +# ----------------------------------------------------------------------------- +# Main +# ----------------------------------------------------------------------------- +if __name__ == "__main__": + start_time = time.time() + + parameters = {} + config_path = os.path.join(os.path.dirname(__file__), "config.yaml") + if os.path.exists(config_path): + with open(config_path, "r") as fh: + parameters = yaml.safe_load(fh) or {} + parameters = parameters or {} + + # ---- sensible defaults / normalization ---- + parameters.setdefault("experiment_name", "ads_ctr_wide_deep") + parameters.setdefault("device", "auto") + parameters.setdefault("training_steps_to_do", 1000000) + parameters.setdefault("eval_full_to_train_steps_ratio", 100) + + exp_name = parameters["experiment_name"] + + wl.watch_or_edit(parameters, flag="hyperparameters", poll_interval=1.0) + + if parameters.get("device", "auto") == "auto": + parameters["device"] = torch.device("cuda" if torch.cuda.is_available() else "cpu") + device = parameters["device"] + + if not parameters.get("root_log_dir"): + parameters["root_log_dir"] = tempfile.mkdtemp() + print(f"No root_log_dir specified, using temporary directory: {parameters['root_log_dir']}") + os.makedirs(parameters["root_log_dir"], exist_ok=True) + + verbose = parameters.get("verbose", True) + log_dir = parameters["root_log_dir"] + tqdm_display = parameters.get("tqdm_display", True) + eval_full_to_train_steps_ratio = parameters.get("eval_full_to_train_steps_ratio", 100) + write_export_ratio = parameters.get("write_export_ratio", 100) + enable_h5_persistence = parameters.get("enable_h5_persistence", True) + training_steps_to_do = parameters.get("training_steps_to_do", 1000) + + # Model (Wide & Deep CTR) + _model = WideDeepCTR( + cardinalities=CATEGORICAL_CARDINALITIES, + num_numeric=NUM_NUMERIC, + emb_dim=parameters.get("model", {}).get("emb_dim", 8), + hidden=parameters.get("model", {}).get("hidden", 128), + num_classes=2, + ).to(device) + model = wl.watch_or_edit(_model, flag="model", device=device) + + # Optimizer + lr = parameters.get("optimizer", {}).get("lr", 0.005) + _optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = wl.watch_or_edit(_optimizer, flag="optimizer") + + # Data (synthetic CTR impressions) — no download needed. + dataset_cfg = parameters.get("dataset", {}) + seed = int(dataset_cfg.get("seed", 0)) + n_train = int(dataset_cfg.get("n_train", 8000)) + n_test = int(dataset_cfg.get("n_test", 2000)) + + train_cfg = parameters.get("data", {}).get("train_loader", {}) + test_cfg = parameters.get("data", {}).get("test_loader", {}) + + _train_dataset = AdsCTRDataset(n_train, seed=seed, max_samples=train_cfg.get("max_samples")) + _test_dataset = AdsCTRDataset(n_test, seed=seed + 1, max_samples=test_cfg.get("max_samples")) + + train_loader = wl.watch_or_edit( + _train_dataset, + flag="data", + loader_name="train_loader", + batch_size=train_cfg.get("batch_size", 64), + shuffle=train_cfg.get("shuffle", True), + is_training=True, + compute_hash=False, + preload_labels=True, + preload_metadata=True, + enable_h5_persistence=enable_h5_persistence, + ) + test_loader = wl.watch_or_edit( + _test_dataset, + flag="data", + loader_name="test_loader", + batch_size=test_cfg.get("batch_size", 256), + shuffle=test_cfg.get("shuffle", False), + is_training=False, + compute_hash=False, + preload_labels=True, + preload_metadata=True, + enable_h5_persistence=enable_h5_persistence, + ) + + # Losses & metrics. Up-weight clicks (~20% prevalence) so the positive class + # drives the gradient. + class_weights = torch.tensor( + parameters.get("class_weights", [1.0, 4.0]), dtype=torch.float32, device=device + ) + train_criterion = wl.watch_or_edit( + nn.CrossEntropyLoss(weight=class_weights, reduction="none"), + flag="loss", signal_name="train-loss-CE", log=True) + test_criterion = wl.watch_or_edit( + nn.CrossEntropyLoss(weight=class_weights, reduction="none"), + flag="loss", signal_name="test-loss-CE", log=True) + + metric = wl.watch_or_edit( + Accuracy(task="multiclass", num_classes=2).to(device), + flag="metric", signal_name="metric-ACC", log=True) + + wl.serve(serving_grpc=parameters.get("serving_grpc", False)) + + print("=" * 60) + print(" STARTING ADS-CTR TRAINING") + print(f" Evaluation every {eval_full_to_train_steps_ratio} steps") + print(f" Dataset splits: train={len(_train_dataset)}, test={len(_test_dataset)}") + print(f" Logs will be saved to: {log_dir}") + print("=" * 60 + "\n") + + if tqdm_display: + train_range = tqdm.tqdm( + range(training_steps_to_do) if training_steps_to_do != None else itertools.count(), + desc="Training", + bar_format="{desc}: {n}/{total} [{elapsed}<{remaining}, {rate_fmt}] {bar} | {postfix}", + ncols=140, + position=0, + leave=True, + ) + else: + train_range = range(training_steps_to_do) if training_steps_to_do != None else itertools.count() + + # ================ + # Training Loop + wl.start_training(timeout=3) + + train_loss = None + test_loss, test_metric = None, None + test_loader_len = len(test_loader) + for train_step in train_range: + age = model.get_age() if hasattr(model, "get_age") else train_step + + train_loss = train(train_loader, model, optimizer, train_criterion, device) + + if age > 0 and age % eval_full_to_train_steps_ratio == 0: + test_loss, test_metric = test( + test_loader, model, test_criterion, metric, device, test_loader_len + ) + + if age > 0 and age % write_export_ratio == 0: + wl.write_history() + wl.write_dataframe() + + if verbose and not tqdm_display: + import sys + msg = f"Step {train_step} (Age {age}): Loss={train_loss:.4f}" + if test_loss is not None: + msg += f" | Test={test_loss:.4f} ({test_metric:.1f}%)" + sys.stdout.write(f"\r{msg:<100}") + sys.stdout.flush() + elif tqdm_display: + postfix_parts = [f"train_loss={train_loss:.4f}"] + if test_loss is not None: + postfix_parts.append(f"test_loss={test_loss:.4f}") + if test_metric is not None: + postfix_parts.append(f"test_acc={test_metric:.1f}%") + train_range.set_postfix_str(" | ".join(postfix_parts)) + + print("\n" + "=" * 60) + print(f" Training completed in {time.time() - start_time:.2f} seconds") + print(f" Logs saved to: {log_dir}") + print("=" * 60) + + wl.write_history() + wl.write_dataframe() + wl.keep_serving() diff --git a/weightslab/examples/PyTorch/wl-ads-recommendation/test_ads_recommendation.py b/weightslab/examples/PyTorch/wl-ads-recommendation/test_ads_recommendation.py new file mode 100644 index 00000000..a6e0c154 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-ads-recommendation/test_ads_recommendation.py @@ -0,0 +1,185 @@ +"""Smoke tests for the ads CTR example (pure PyTorch, no weightslab). + +Run: python -m pytest test_ads_recommendation.py + or: python test_ads_recommendation.py +""" + +import os +import sys + +import numpy as np +import torch + +sys.path.insert(0, os.path.dirname(__file__)) + +from utils.data import ( # noqa: E402 + CATEGORICAL_CARDINALITIES, + CATEGORICAL_FIELDS, + IMG_SIDE, + NUM_CATEGORICAL, + NUM_FIELDS, + NUM_NUMERIC, + NUMERIC_FIELDS, + AdsCTRDataset, + category_label, + make_synthetic_ctr, + unpack, +) +from utils.model import WideDeepCTR # noqa: E402 + + +def test_schema(): + assert NUM_FIELDS == 16 == IMG_SIDE * IMG_SIDE + assert NUM_CATEGORICAL == len(CATEGORICAL_FIELDS) == len(CATEGORICAL_CARDINALITIES) == 8 + assert NUM_NUMERIC == 8 + + +def test_synthetic_shapes_and_ranges(): + cat, num_std, num_raw, y = make_synthetic_ctr(500, seed=0) + assert cat.shape == (500, NUM_CATEGORICAL) + assert num_std.shape == (500, NUM_NUMERIC) + assert num_raw.shape == (500, NUM_NUMERIC) + assert y.shape == (500,) + assert str(cat.dtype) == "int64" and str(num_std.dtype) == "float32" and str(num_raw.dtype) == "float32" + for f, card in enumerate(CATEGORICAL_CARDINALITIES): + assert cat[:, f].min() >= 0 and cat[:, f].max() < card + assert set(int(v) for v in set(y.tolist())) <= {0, 1} + + +def test_numeric_standardized_vs_raw(): + """Model input (standardized) must differ in scale from the raw metadata values.""" + _, num_std, num_raw, _ = make_synthetic_ctr(2000, seed=0) + # Standardized: ~zero mean, ~unit variance per feature. + assert np.allclose(num_std.mean(axis=0), 0.0, atol=0.15) + assert np.allclose(num_std.std(axis=0), 1.0, atol=0.15) + # Raw values live at their natural human-readable scale (not standardized), + # e.g. bid_price in dollars, user_age in years — nowhere near unit variance. + bid_idx = NUMERIC_FIELDS.index("bid_price") + age_idx = NUMERIC_FIELDS.index("user_age") + assert num_raw[:, bid_idx].mean() > 1.0 # dollars, not ~0 + assert num_raw[:, age_idx].mean() > 18.0 # years, not ~0 + assert not np.allclose(num_std, num_raw) + + +def test_deterministic_for_seed(): + a = make_synthetic_ctr(200, seed=5) + b = make_synthetic_ctr(200, seed=5) + assert all((x == y).all() for x, y in zip(a, b)) + c = make_synthetic_ctr(200, seed=6) + assert not (a[0] == c[0]).all() + + +def test_ctr_is_realistic(): + _, _, _, y = make_synthetic_ctr(4000, seed=1) + assert 0.15 < float(y.mean()) < 0.26 # calibrated to ~20% CTR + + +def test_dataset_item_contract(): + ds = AdsCTRDataset(300, seed=2) + assert len(ds) == 300 + x, idx, label = ds[0] + # Model input is the packed 1-D field vector (no fake image). + assert tuple(x.shape) == (NUM_FIELDS,) + assert x.dtype == torch.float32 + assert idx == 0 and label in (0, 1) + + +def test_unpack_roundtrip(): + ds = AdsCTRDataset(64, seed=3) + image = torch.stack([ds[i][0] for i in range(len(ds))]) # [N,1,4,4] + cat, num = unpack(image) + assert cat.shape == (len(ds), NUM_CATEGORICAL) + assert num.shape == (len(ds), NUM_NUMERIC) + assert (cat == ds.cat).all(), "categorical indices must survive pack/unpack" + assert torch.allclose(num, ds.num, atol=1e-5) + + +def test_get_items_exposes_field_metadata_columns(): + """The ledger-init contract must expose every field as a metadata column.""" + ds = AdsCTRDataset(50, seed=2) + image, uid, target, metadata = ds.get_items( + 4, include_metadata=True, include_labels=True, include_images=False + ) + assert image is None + assert uid == 4 and target in (0, 1) + assert set(metadata.keys()) == set(CATEGORICAL_FIELDS) | set(NUMERIC_FIELDS) + # Categorical fields are readable strings, numeric are floats. + assert isinstance(metadata["placement"], str) + assert metadata["device_type"] in ("mobile", "desktop", "tablet", "ctv") + assert all(isinstance(metadata[n], float) for n in NUMERIC_FIELDS) + + +def test_metadata_numeric_values_are_raw_not_standardized(): + """Metadata must carry the raw (human-scale) values, not the standardized + model input — the UI shows the model input in list mode and the raw + values as metadata, so the two must not collapse to the same numbers.""" + ds = AdsCTRDataset(50, seed=2) + _, _, _, metadata = ds.get_items(4, include_metadata=True) + for j, name in enumerate(NUMERIC_FIELDS): + assert metadata[name] == round(float(ds.num_raw[4][j]), 4) + assert metadata[name] != round(float(ds.num[4][j]), 4) + + +def test_category_label_fallback(): + assert category_label("device_type", 0) == "mobile" + assert category_label("region", 3) == "region_3" # no vocab -> fallback + + +def test_model_forward_shape(): + model = WideDeepCTR() + assert model(torch.randn(8, NUM_FIELDS)).shape == (8, 2) + assert model(torch.randn(8, 1, IMG_SIDE, IMG_SIDE)).shape == (8, 2) + + +def test_training_learns_to_rank_clicks(): + """Training must reduce loss and rank real clicks above non-clicks on a + held-out split (train/test share the same ground-truth CTR model).""" + torch.manual_seed(0) + train_ds = AdsCTRDataset(4000, seed=0) + test_ds = AdsCTRDataset(1000, seed=1) + + train_x = torch.stack([train_ds[i][0] for i in range(len(train_ds))]) + train_y = train_ds.labels + test_x = torch.stack([test_ds[i][0] for i in range(len(test_ds))]) + test_y = test_ds.labels + + model = WideDeepCTR() + optimizer = torch.optim.Adam(model.parameters(), lr=5e-3) + # Up-weight the ~20% positive class. + criterion = torch.nn.CrossEntropyLoss(weight=torch.tensor([1.0, 4.0])) + + model.train() + initial_loss = criterion(model(train_x), train_y).item() + batch = 256 + for _ in range(30): + perm = torch.randperm(len(train_ds)) + for s in range(0, len(train_ds), batch): + idx = perm[s:s + batch] + optimizer.zero_grad() + loss = criterion(model(train_x[idx]), train_y[idx]) + loss.backward() + optimizer.step() + final_loss = criterion(model(train_x), train_y).item() + assert final_loss < initial_loss, f"loss did not drop: {initial_loss:.4f} -> {final_loss:.4f}" + + # Ranking quality on the held-out split: P(click) higher for real clicks. + model.eval() + with torch.no_grad(): + prob_click = torch.softmax(model(test_x), dim=1)[:, 1] + pos = prob_click[test_y == 1].mean().item() + neg = prob_click[test_y == 0].mean().item() + assert pos > neg + 0.05, f"model does not separate clicks: pos={pos:.3f} neg={neg:.3f}" + + # Rank-AUC (Mann-Whitney) should beat random (0.5). + with torch.no_grad(): + p = prob_click + pos_p = p[test_y == 1] + neg_p = p[test_y == 0] + wins = (pos_p.unsqueeze(1) > neg_p.unsqueeze(0)).float().mean().item() + assert wins > 0.6, f"AUC too low: {wins:.3f}" + + +if __name__ == "__main__": + import pytest + + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/weightslab/examples/PyTorch/ws-clustering/face/__init__.py b/weightslab/examples/PyTorch/wl-ads-recommendation/utils/__init__.py similarity index 100% rename from weightslab/examples/PyTorch/ws-clustering/face/__init__.py rename to weightslab/examples/PyTorch/wl-ads-recommendation/utils/__init__.py diff --git a/weightslab/examples/PyTorch/wl-ads-recommendation/utils/data.py b/weightslab/examples/PyTorch/wl-ads-recommendation/utils/data.py new file mode 100644 index 00000000..b71b3823 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-ads-recommendation/utils/data.py @@ -0,0 +1,269 @@ +"""Synthetic advertising click-through-rate (CTR) dataset (pure PyTorch). + +Import-light (only ``numpy`` + ``torch``) so it can be unit-tested without +importing ``weightslab`` — see ``test_ads_recommendation.py``. + +CTR prediction is the core of an advertising recommendation system: given a +(user, ad, context) triple, predict P(click). This module is a reproducible, +offline stand-in for that task. Each impression has **8 categorical fields** +(user segment, ad category, device, OS, publisher, placement, region, hour +bucket) and **8 numeric features** (ad position, bid, user age, session depth, +historical CTR, …), with a binary ``clicked`` label at a realistic ~20% CTR. + +Real-world analogues (drop-in replaceable): + * Criteo Display Advertising CTR — 13 numeric + 26 categorical fields. + + * Avazu CTR — + * MovieLens (for the recommender variant) — + +Canonical models for this task: Wide & Deep, DeepFM, Factorization Machines. +``utils/model.py`` implements a compact Wide & Deep. + +The model input is the 16-field vector itself (categorical indices + per-feature +standardized numerics; no fake image). WeightsLab transmits it through gRPC as a +``vector`` raw_data stat carrying the actual values, and ``get_items`` exposes the +raw, human-readable field values (dollars, years, page counts, …) as sortable +metadata columns in the List Exploration (tabular) view. +""" + +from __future__ import annotations + +from typing import List, Optional, Tuple + +import numpy as np +import torch +from torch.utils.data import Dataset + +# --- Field schema --------------------------------------------------------- +CATEGORICAL_FIELDS = [ + "user_segment", + "ad_category", + "device_type", + "os", + "publisher", + "placement", + "region", + "hour_bucket", +] +CATEGORICAL_CARDINALITIES = [6, 10, 4, 4, 12, 5, 8, 6] + +NUMERIC_FIELDS = [ + "ad_position", + "bid_price", + "user_age", + "session_depth", + "historical_ctr", + "days_since_last_click", + "num_ads_seen_today", + "creative_freshness", +] + +NUM_CATEGORICAL = len(CATEGORICAL_FIELDS) # 8 +NUM_NUMERIC = len(NUMERIC_FIELDS) # 8 +NUM_FIELDS = NUM_CATEGORICAL + NUM_NUMERIC # 16 +IMG_SIDE = 4 +assert IMG_SIDE * IMG_SIDE == NUM_FIELDS + +# Human-readable labels per categorical field (len == cardinality), used only to +# make the UI metadata columns legible; the model still sees integer codes. +CATEGORICAL_VOCABS = { + "device_type": ["mobile", "desktop", "tablet", "ctv"], + "os": ["android", "ios", "windows", "other"], + "placement": ["banner", "sidebar", "interstitial", "native", "video"], + "hour_bucket": ["night", "early", "morning", "midday", "afternoon", "evening"], +} + + +def category_label(field: str, code: int) -> str: + """Readable label for a categorical code, or ``"_"`` fallback.""" + vocab = CATEGORICAL_VOCABS.get(field) + if vocab is not None and 0 <= code < len(vocab): + return vocab[code] + return f"{field}_{code}" + +TARGET_CTR = 0.20 +# Fixed seed for the *ground-truth* click model, independent of the data seed, +# so train and test splits share the same underlying mapping. +_TRUE_PARAM_SEED = 12345 + + +def _true_params(): + """Stable 'ground-truth' click model parameters (same across all splits).""" + rng = np.random.default_rng(_TRUE_PARAM_SEED) + cat_weights = [rng.normal(0.0, 0.8, size=card) for card in CATEGORICAL_CARDINALITIES] + numeric_weights = rng.normal(0.0, 0.5, size=NUM_NUMERIC) + # One pairwise interaction: ad_category x user_segment (classic in ad CTR). + interaction = rng.normal( + 0.0, 0.7, size=(CATEGORICAL_CARDINALITIES[1], CATEGORICAL_CARDINALITIES[0]) + ) + return cat_weights, numeric_weights, interaction + + +def _draw_raw_numeric(rng: np.random.Generator, n: int) -> np.ndarray: + """Draw the 8 numeric fields at their natural, human-readable scale. + + Order matches ``NUMERIC_FIELDS``. These are the values an ad-serving + pipeline would actually log (dollars, years, page counts, …) — this is + what the UI shows as metadata; the model never sees these directly. + """ + ad_position = rng.integers(1, 9, size=n).astype(np.float64) # slot 1-8 + bid_price = rng.gamma(2.0, 1.25, size=n) # dollars, mean ~$2.50 CPC + user_age = np.clip(rng.normal(35.0, 12.0, size=n), 18, 80) + session_depth = rng.poisson(3.0, size=n).astype(np.float64) + 1 # pages viewed + historical_ctr = rng.beta(2.0, 25.0, size=n) # user's baseline CTR, mean ~7% + days_since_last_click = rng.exponential(10.0, size=n) + num_ads_seen_today = rng.poisson(12.0, size=n).astype(np.float64) + 1 + creative_freshness = rng.exponential(20.0, size=n) # days since creative was made + + return np.stack( + [ + ad_position, bid_price, user_age, session_depth, + historical_ctr, days_since_last_click, num_ads_seen_today, + creative_freshness, + ], + axis=1, + ) + + +def make_synthetic_ctr( + n_samples: int, seed: int = 0 +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Generate deterministic synthetic ad impressions. + + Returns ``(cat, num_std, num_raw, y)``: + * ``cat``: ``int64[n, NUM_CATEGORICAL]`` category indices + * ``num_std``: ``float32[n, NUM_NUMERIC]`` standardized numeric features + (model input) + * ``num_raw``: ``float32[n, NUM_NUMERIC]`` raw, human-readable numeric + values (surfaced as sortable metadata columns in the UI) + * ``y``: ``int64[n]`` click label (1 = clicked), ~20% positive + """ + if n_samples <= 0: + empty_num = np.zeros((0, NUM_NUMERIC), dtype=np.float32) + return ( + np.zeros((0, NUM_CATEGORICAL), dtype=np.int64), + empty_num, + empty_num.copy(), + np.zeros((0,), dtype=np.int64), + ) + + rng = np.random.default_rng(seed) + cat_weights, numeric_weights, interaction = _true_params() + + # Sample categorical indices (uniform over each field's cardinality). + cat = np.stack( + [rng.integers(0, card, size=n_samples) for card in CATEGORICAL_CARDINALITIES], + axis=1, + ).astype(np.int64) + + # Raw, human-readable numeric values, then standardize per-feature + # (class-agnostic) for the model input — mirrors the fraud-detection example. + num_raw = _draw_raw_numeric(rng, n_samples) + mean = num_raw.mean(axis=0, keepdims=True) + std = num_raw.std(axis=0, keepdims=True) + std[std == 0] = 1.0 + num_std = (num_raw - mean) / std + + # Ground-truth click logit = sum of per-field effects + one interaction + numeric. + logit = np.zeros(n_samples, dtype=np.float64) + for f, w in enumerate(cat_weights): + logit += w[cat[:, f]] + logit += num_std @ numeric_weights + logit += interaction[cat[:, 1], cat[:, 0]] # ad_category x user_segment + + # Calibrate an additive bias so the *mean click probability* lands near + # TARGET_CTR. Centering the logit mean is not enough (mean(sigmoid) != + # sigmoid(mean) by Jensen), so binary-search the bias on the actual mean prob. + logit -= logit.mean() + lo, hi = -20.0, 20.0 + for _ in range(60): + mid = 0.5 * (lo + hi) + mean_prob = (1.0 / (1.0 + np.exp(-(logit + mid)))).mean() + if mean_prob < TARGET_CTR: + lo = mid + else: + hi = mid + logit += 0.5 * (lo + hi) + + prob = 1.0 / (1.0 + np.exp(-logit)) + y = (rng.uniform(0.0, 1.0, size=n_samples) < prob).astype(np.int64) + + return cat, num_std.astype(np.float32), num_raw.astype(np.float32), y + + +class AdsCTRDataset(Dataset): + """Tabular ads CTR dataset yielding ``(input, idx, label)``. + + ``input`` is the 1-D packed field vector ``float32[NUM_FIELDS]`` fed straight + to the model — the 8 categorical indices (as floats) followed by the 8 + per-feature standardized numerics; there is no image. ``idx`` is the tracked + sample id; ``label`` is 0 (no click) / 1 (click). + """ + + def __init__(self, n_samples: int, seed: int = 0, max_samples: Optional[int] = None): + cat, num_std, num_raw, y = make_synthetic_ctr(n_samples, seed=seed) + if max_samples is not None: + cat, num_std, num_raw, y = ( + cat[:max_samples], num_std[:max_samples], num_raw[:max_samples], y[:max_samples], + ) + packed = np.concatenate([cat.astype(np.float32), num_std], axis=1) # [N, 16] (model input) + self.features = torch.from_numpy(packed) # float32 [N, 16] (standardized, model input) + self.cat = torch.from_numpy(cat) # int64 [N, 8] + self.num = torch.from_numpy(num_std) # float32 [N, 8] (standardized) + self.num_raw = num_raw # float32 [N, 8] (raw, display values) + self.labels = torch.from_numpy(y) # int64 [N] + + def __len__(self) -> int: + return int(self.features.shape[0]) + + def _input(self, idx: int) -> torch.Tensor: + # Tabular: the model input IS the packed 1-D field vector (no fake image). + return self.features[idx] + + def _metadata(self, idx: int) -> dict: + """Readable category labels + raw (non-standardized) numeric values -> sortable UI columns.""" + meta = {} + cat_row = self.cat[idx] + for f, name in enumerate(CATEGORICAL_FIELDS): + meta[name] = category_label(name, int(cat_row[f])) + raw_row = self.num_raw[idx] + for j, name in enumerate(NUMERIC_FIELDS): + meta[name] = round(float(raw_row[j]), 4) + return meta + + def __getitems__(self, idx: int): + # Training contract: (input, sample_id, label). + return self._input(idx), idx, int(self.labels[idx].item()) + + def __getitem__(self, idx: int): + # Training contract: (input, sample_id, label). + return self._input(idx), idx, int(self.labels[idx].item()) + + def get_items(self, idx: int, include_metadata: bool = False, + include_labels: bool = False, include_images: bool = False): + """WeightsLab ledger-init contract: (image, uid, target, metadata). + + The returned ``metadata`` dict (readable categorical labels + numeric + features) is flattened into per-sample columns, so each impression field + (``ad_category``, ``placement``, ``bid_price``, …) becomes a sortable + column in the List Exploration view. + """ + image = self._input(idx) if include_images else None + target = int(self.labels[idx].item()) if include_labels else None + metadata = self._metadata(idx) if include_metadata else None + return image, idx, target, metadata + + +def unpack(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Split a packed ``[N, 16]`` (or ``[N, 1, 4, 4]``) batch into (cat_idx, numeric). + + ``cat_idx`` is ``long[N, 8]`` clamped to valid ranges; ``numeric`` is + ``float[N, 8]``. Shared by the model and the tests so packing lives in one + place. + """ + flat = x.reshape(x.shape[0], -1) + cat = flat[:, :NUM_CATEGORICAL].round().long() + num = flat[:, NUM_CATEGORICAL:] + card = torch.tensor(CATEGORICAL_CARDINALITIES, device=x.device) + cat = cat.clamp(min=torch.zeros_like(card), max=card - 1) + return cat, num diff --git a/weightslab/examples/PyTorch/wl-ads-recommendation/utils/model.py b/weightslab/examples/PyTorch/wl-ads-recommendation/utils/model.py new file mode 100644 index 00000000..dfef847d --- /dev/null +++ b/weightslab/examples/PyTorch/wl-ads-recommendation/utils/model.py @@ -0,0 +1,76 @@ +"""Wide & Deep CTR model for advertising recommendation (pure PyTorch). + +A compact take on the classic Wide & Deep architecture (Cheng et al., 2016): + * **Deep**: each categorical field -> embedding; embeddings are concatenated + with the numeric features and passed through an MLP (learns generalizable + feature interactions). + * **Wide**: first-order per-category effects + a linear numeric term (memorizes + strong direct signals). +The two logits are summed. Outputs 2 logits (no-click / click) so it plugs into +``CrossEntropyLoss`` and a 2-class accuracy metric like the other usecases. +""" + +from __future__ import annotations + +from typing import List + +import torch +import torch.nn as nn + +from .data import ( + CATEGORICAL_CARDINALITIES, + NUM_CATEGORICAL, + NUM_NUMERIC, + IMG_SIDE, + unpack, +) + + +class WideDeepCTR(nn.Module): + def __init__( + self, + cardinalities: List[int] = CATEGORICAL_CARDINALITIES, + num_numeric: int = NUM_NUMERIC, + emb_dim: int = 8, + hidden: int = 128, + num_classes: int = 2, + ): + super().__init__() + self.input_shape = (1, len(cardinalities) + num_numeric) + self.cardinalities = list(cardinalities) + self.num_numeric = num_numeric + + # Deep: one embedding table per categorical field. + self.embeddings = nn.ModuleList( + [nn.Embedding(card, emb_dim) for card in self.cardinalities] + ) + deep_in = emb_dim * len(self.cardinalities) + num_numeric + self.deep = nn.Sequential( + nn.Linear(deep_in, hidden), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden, hidden // 2), + nn.ReLU(), + nn.Linear(hidden // 2, num_classes), + ) + + # Wide: first-order per-category effect (embedding dim 1) + linear numeric. + self.wide_cat = nn.ModuleList( + [nn.Embedding(card, num_classes) for card in self.cardinalities] + ) + self.wide_num = nn.Linear(num_numeric, num_classes) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + cat_idx, numeric = unpack(x) # [N, 8] long, [N, 8] float + + # Deep branch. + embs = [emb(cat_idx[:, i]) for i, emb in enumerate(self.embeddings)] + deep_in = torch.cat(embs + [numeric], dim=1) + deep_logits = self.deep(deep_in) + + # Wide branch. + wide_logits = self.wide_num(numeric) + for i, emb in enumerate(self.wide_cat): + wide_logits = wide_logits + emb(cat_idx[:, i]) + + return deep_logits + wide_logits diff --git a/weightslab/examples/PyTorch/wl-ads-recommendation/verify_integration.py b/weightslab/examples/PyTorch/wl-ads-recommendation/verify_integration.py new file mode 100644 index 00000000..b8b08d01 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-ads-recommendation/verify_integration.py @@ -0,0 +1,219 @@ +"""Integration check: prove WeightsLab is fully wired for this tabular example. + +Unlike ``test_ads_recommendation.py`` (pure PyTorch), this drives the WeightsLab +stack end-to-end — tracked dataloaders, watched loss/metric, the gRPC server, +and the per-sample ledger — then asserts the sample dataframe the UI reads +contains, for tabular mode: + + * one row per impression (what you scroll/sort in the grid & List view), + * every ad/user/context field as its own column (from ``get_items`` metadata), + * ``target`` + ``prediction`` populated (so sort/filter/histograms work), + * a per-sample training/eval loss column, + +and that the gRPC server is actually listening (so Weights Studio can connect +live during the run). + +Run: python verify_integration.py +Requires weightslab installed (not a pure-unit test). +""" + +import glob +import os +import socket +import sys +import tempfile +import time as _time + +import torch +import torch.nn as nn +import torch.optim as optim +from torchmetrics.classification import Accuracy + +sys.path.insert(0, os.path.dirname(__file__)) + +import weightslab as wl # noqa: E402 +from weightslab.components.global_monitoring import ( # noqa: E402 + guard_training_context, + guard_testing_context, +) + +from utils.data import ( # noqa: E402 + AdsCTRDataset, + CATEGORICAL_CARDINALITIES, + CATEGORICAL_FIELDS, + NUMERIC_FIELDS, + NUM_NUMERIC, +) +from utils.model import WideDeepCTR # noqa: E402 + + +def _free_port() -> int: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +def main() -> int: + device = torch.device("cpu") + log_dir = tempfile.mkdtemp(prefix="wl_ads_verify_") + grpc_port = _free_port() + os.environ["GRPC_BACKEND_PORT"] = str(grpc_port) + + parameters = { + "experiment_name": "ads_verify", + "device": "cpu", + "root_log_dir": log_dir, + "serving_grpc": True, + "grpc_port": grpc_port, + "is_training": True, + } + wl.watch_or_edit(parameters, flag="hyperparameters", poll_interval=1.0) + + model = wl.watch_or_edit( + WideDeepCTR(cardinalities=CATEGORICAL_CARDINALITIES, num_numeric=NUM_NUMERIC).to(device), + flag="model", device=device) + optimizer = wl.watch_or_edit(optim.Adam(model.parameters(), lr=0.005), flag="optimizer") + + train_ds = AdsCTRDataset(800, seed=0) + test_ds = AdsCTRDataset(200, seed=1) + + train_loader = wl.watch_or_edit( + train_ds, flag="data", loader_name="train_loader", batch_size=64, shuffle=True, + is_training=True, compute_hash=False, preload_labels=True, preload_metadata=True, + enable_h5_persistence=False) + test_loader = wl.watch_or_edit( + test_ds, flag="data", loader_name="test_loader", batch_size=128, shuffle=False, + is_training=False, compute_hash=False, preload_labels=True, preload_metadata=True, + enable_h5_persistence=False) + + cw = torch.tensor([1.0, 4.0]) + train_crit = wl.watch_or_edit(nn.CrossEntropyLoss(weight=cw, reduction="none"), + flag="loss", signal_name="train-loss-CE", log=True) + test_crit = wl.watch_or_edit(nn.CrossEntropyLoss(weight=cw, reduction="none"), + flag="loss", signal_name="test-loss-CE", log=True) + metric = wl.watch_or_edit(Accuracy(task="multiclass", num_classes=2), + flag="metric", signal_name="metric-ACC", log=True) + + wl.serve(serving_grpc=True, grpc_port=grpc_port) + wl.start_training() + + for _ in range(120): + with guard_training_context: + inputs, ids, labels = next(train_loader) + optimizer.zero_grad() + out = model(inputs) + preds = out.argmax(dim=1, keepdim=True) + loss = train_crit(out, labels, batch_ids=ids, preds=preds).mean() + loss.backward() + optimizer.step() + + for inputs, ids, labels in test_loader: + with guard_testing_context: + out = model(inputs) + preds = out.argmax(dim=1, keepdim=True) + test_crit(out, labels, batch_ids=ids, preds=preds) + metric.update(out, labels) + + wl.drain_signals() + + wl.write_dataframe(path=log_dir, format="csv") + csvs = glob.glob(os.path.join(log_dir, "*dataframe*.csv")) + assert csvs, f"no dataframe CSV written to {log_dir}" + with open(csvs[0], "r", encoding="utf-8") as fh: + header = fh.readline().strip().split(",") + rows = [ln for ln in fh.read().splitlines() if ln.strip()] + + cols = set(c.strip().strip('"') for c in header) + n_rows = len(rows) + + print("\n=== Ledger dataframe ===") + print(f"rows: {n_rows}") + print(f"columns ({len(cols)}): {sorted(cols)}") + + problems = [] + if n_rows < 900: + problems.append(f"expected ~1000 sample rows, got {n_rows}") + + field_cols = set(CATEGORICAL_FIELDS) | set(NUMERIC_FIELDS) + missing_fields = [f for f in field_cols if f not in cols] + if missing_fields: + problems.append(f"missing field columns: {missing_fields}") + + for required in ("target", "prediction"): + if required not in cols: + problems.append(f"missing '{required}' column") + + if not any("loss" in c.lower() for c in cols): + problems.append("no per-sample loss column found") + + grpc_up = False + for _ in range(20): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(1) + if sock.connect_ex(("127.0.0.1", grpc_port)) == 0: + sock.close() + grpc_up = True + break + sock.close() + _time.sleep(0.5) + if not grpc_up: + problems.append(f"gRPC server not listening on {grpc_port}") + else: + print(f"gRPC server listening on 127.0.0.1:{grpc_port} OK") + + # Real gRPC round-trip: the packed field vector fed to the model must + # reach the UI as raw_data of type 'vector' carrying the values. + try: + import grpc + from weightslab.proto import experiment_service_pb2 as pb2 + from weightslab.proto import experiment_service_pb2_grpc as pb2_grpc + + n_fields = len(CATEGORICAL_CARDINALITIES) + NUM_NUMERIC + channel = grpc.insecure_channel(f"127.0.0.1:{grpc_port}") + grpc.channel_ready_future(channel).result(timeout=10) + stub = pb2_grpc.ExperimentServiceStub(channel) + resp = stub.GetDataSamples(pb2.DataSamplesRequest( + start_index=0, records_cnt=5, include_raw_data=True, + resize_width=0, resize_height=0), timeout=20) + + raw = None + for rec in resp.data_records: + for st in rec.data_stats: + if st.name == "raw_data": + raw = st + break + if raw is not None: + break + + if raw is None: + problems.append("gRPC GetDataSamples returned no raw_data stat") + elif raw.type != "vector": + problems.append(f"raw_data.type is '{raw.type}', expected 'vector'") + elif len(raw.value) != n_fields: + problems.append( + f"raw_data carries {len(raw.value)} values, expected {n_fields}") + else: + print(f"gRPC GetDataSamples: raw_data type='vector', " + f"{len(raw.value)} field values reached the UI OK") + channel.close() + except Exception as e: + problems.append(f"gRPC GetDataSamples failed: {e}") + + if problems: + print("\n VERIFY FAILED:") + for p in problems: + print(f" - {p}") + return 1 + + print("\n VERIFY PASSED: WeightsLab is fully wired for tabular mode " + "(per-sample rows + field columns + target/prediction/loss + live gRPC).") + return 0 + + +if __name__ == "__main__": + code = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(code) diff --git a/weightslab/examples/PyTorch/ws-classification/config.yaml b/weightslab/examples/PyTorch/wl-classification/config.yaml similarity index 100% rename from weightslab/examples/PyTorch/ws-classification/config.yaml rename to weightslab/examples/PyTorch/wl-classification/config.yaml diff --git a/weightslab/examples/PyTorch/ws-classification/main.py b/weightslab/examples/PyTorch/wl-classification/main.py similarity index 100% rename from weightslab/examples/PyTorch/ws-classification/main.py rename to weightslab/examples/PyTorch/wl-classification/main.py diff --git a/weightslab/examples/PyTorch/ws-clustering/config.yaml b/weightslab/examples/PyTorch/wl-clustering/config.yaml similarity index 100% rename from weightslab/examples/PyTorch/ws-clustering/config.yaml rename to weightslab/examples/PyTorch/wl-clustering/config.yaml diff --git a/weightslab/examples/PyTorch/wl-clustering/face/__init__.py b/weightslab/examples/PyTorch/wl-clustering/face/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/weightslab/examples/PyTorch/ws-clustering/face/data.py b/weightslab/examples/PyTorch/wl-clustering/face/data.py similarity index 100% rename from weightslab/examples/PyTorch/ws-clustering/face/data.py rename to weightslab/examples/PyTorch/wl-clustering/face/data.py diff --git a/weightslab/examples/PyTorch/ws-clustering/face/model.py b/weightslab/examples/PyTorch/wl-clustering/face/model.py similarity index 100% rename from weightslab/examples/PyTorch/ws-clustering/face/model.py rename to weightslab/examples/PyTorch/wl-clustering/face/model.py diff --git a/weightslab/examples/PyTorch/ws-clustering/face/signals.py b/weightslab/examples/PyTorch/wl-clustering/face/signals.py similarity index 100% rename from weightslab/examples/PyTorch/ws-clustering/face/signals.py rename to weightslab/examples/PyTorch/wl-clustering/face/signals.py diff --git a/weightslab/examples/PyTorch/ws-clustering/face/utils.py b/weightslab/examples/PyTorch/wl-clustering/face/utils.py similarity index 100% rename from weightslab/examples/PyTorch/ws-clustering/face/utils.py rename to weightslab/examples/PyTorch/wl-clustering/face/utils.py diff --git a/weightslab/examples/PyTorch/ws-clustering/main.py b/weightslab/examples/PyTorch/wl-clustering/main.py similarity index 100% rename from weightslab/examples/PyTorch/ws-clustering/main.py rename to weightslab/examples/PyTorch/wl-clustering/main.py diff --git a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/README.md b/weightslab/examples/PyTorch/wl-detection/README.md similarity index 99% rename from weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/README.md rename to weightslab/examples/PyTorch/wl-detection/README.md index ba8bddc6..3fee76a9 100644 --- a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/README.md +++ b/weightslab/examples/PyTorch/wl-detection/README.md @@ -20,7 +20,7 @@ weightslab start example --det Or run it directly: ```bash -cd weightslab/examples/PyTorch/ws-detection +cd weightslab/examples/PyTorch/wl-detection pip install -r requirements.txt python main.py ``` diff --git a/weightslab/examples/PyTorch/ws-detection/config.yaml b/weightslab/examples/PyTorch/wl-detection/config.yaml similarity index 100% rename from weightslab/examples/PyTorch/ws-detection/config.yaml rename to weightslab/examples/PyTorch/wl-detection/config.yaml diff --git a/weightslab/examples/PyTorch/ws-detection/main.py b/weightslab/examples/PyTorch/wl-detection/main.py similarity index 99% rename from weightslab/examples/PyTorch/ws-detection/main.py rename to weightslab/examples/PyTorch/wl-detection/main.py index ff4b2030..6cb4d627 100644 --- a/weightslab/examples/PyTorch/ws-detection/main.py +++ b/weightslab/examples/PyTorch/wl-detection/main.py @@ -119,6 +119,8 @@ def test(loader, model, sig, device, grid_size, conf_thresh, test_loader_len): parameters.setdefault("freeze_backbone", True) parameters.setdefault("compute_natural_sort", True) + exp_name = parameters["experiment_name"] + # --- 2) Register hyperparameters --- exp_name = parameters["experiment_name"] wl.watch_or_edit( @@ -128,7 +130,6 @@ def test(loader, model, sig, device, grid_size, conf_thresh, test_loader_len): defaults=parameters, poll_interval=1.0, ) - num_classes = int(parameters["num_classes"]) image_size = int(parameters["image_size"]) grid_size = int(parameters["grid_size"]) diff --git a/weightslab/examples/PyTorch/ws-detection/utils/criterions.py b/weightslab/examples/PyTorch/wl-detection/utils/criterions.py similarity index 100% rename from weightslab/examples/PyTorch/ws-detection/utils/criterions.py rename to weightslab/examples/PyTorch/wl-detection/utils/criterions.py diff --git a/weightslab/examples/PyTorch/ws-detection/utils/data.py b/weightslab/examples/PyTorch/wl-detection/utils/data.py similarity index 100% rename from weightslab/examples/PyTorch/ws-detection/utils/data.py rename to weightslab/examples/PyTorch/wl-detection/utils/data.py diff --git a/weightslab/examples/PyTorch/ws-detection/utils/model.py b/weightslab/examples/PyTorch/wl-detection/utils/model.py similarity index 100% rename from weightslab/examples/PyTorch/ws-detection/utils/model.py rename to weightslab/examples/PyTorch/wl-detection/utils/model.py diff --git a/weightslab/examples/PyTorch/wl-fraud-detection/README.md b/weightslab/examples/PyTorch/wl-fraud-detection/README.md new file mode 100644 index 00000000..d32a00a3 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-fraud-detection/README.md @@ -0,0 +1,92 @@ +# WeightsLab — Bank Fraud Detection (tabular, pure PyTorch) + +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GrayboxTech/weightslab/blob/main/weightslab/examples/Notebooks/PyTorch/wl-fraud-detection.ipynb) + +A small, fully-runnable **tabular binary-classification** example: an MLP learns +to flag fraudulent bank card transactions, streaming per-sample loss / +prediction / accuracy to the WeightsLab UI. Being tabular, it's the natural +companion to the **List Exploration (tabular) view** — sort by `loss` or +`prediction` to triage the transactions the model finds hardest. + +Everything is plain PyTorch + NumPy — no external download. The dataset is +generated in-process so runs are reproducible and offline. + +## Quick start + +```bash +cd weightslab/examples/PyTorch/wl-fraud-detection +pip install -r requirements.txt +python main.py +``` + +Then open the UI (e.g. `http://localhost:5173`), switch the Data Exploration +board to **List** view, press play, and sort columns to explore. + +## The data + +`make_synthetic_fraud(n, seed)` (in `utils/data.py`) generates a reproducible +stream of transactions. Each row has **16 numeric features** (`amount`, +`old_balance`, `merchant_risk`, `geo_distance_km`, `is_foreign`, +`num_prior_disputes`, …) and a binary label (`0` legit, `1` fraud, ~12% +prevalence). Fraud rows come from shifted distributions (larger amounts, +off-hours activity, higher merchant risk, device changes, larger geo jumps). +Features are standardized and reshaped to a `1x4x4` heatmap so the grid renders +a thumbnail; the model flattens it back to 16. + +### Using a real dataset + +Swap `FraudDataset` for a loader over a real CSV to go from demo to reality: + +* **Kaggle Credit Card Fraud** (ULB) — `creditcard.csv`, 284k transactions with + anonymized `V1..V28` features + `Amount` + `Class`. + +* **PaySim** mobile-money fraud simulator. + + +Keep the `__getitem__` contract `(features_as_image, idx, label)` and adjust +`NUM_FEATURES` / the reshape. + +## What "a sample" is here + +There are no images — **each sample is one transaction (a row)**, and the model +input **is** the 1-D feature vector (not a reshaped image). WeightsLab carries +that vector through gRPC as a `raw_data` stat of type `vector` (the actual +values), so `inputs`, `labels`/`target` and `metadata` all reach the UI. The 16 +raw features are also exposed as **sortable columns** via the dataset's +`get_items()` metadata contract (`preload_metadata=True`), so the List +Exploration view shows real tabular columns (`amount`, `merchant_risk`, +`geo_distance_km`, …) alongside the tracked stats below. Everything you do on +MNIST — sort, lock, histograms, discard/restore, neuron ops — works the same +way, because those operate on the per-sample ledger, not on pixels. + +## What you'll see in the UI + +| Signal / column | Meaning | +| ---------------------------------------- | ---------------------------------------------- | +| feature columns (`amount`, `merchant_risk`, …) | The 16 raw transaction features, sortable | +| `train-loss-CE`, `test-loss-CE` | Weighted cross-entropy per split | +| `metric-ACC` | Overall accuracy | +| `test_metric/Accuracy_per_sample` | Per-transaction correctness (0/1) | +| `test_metric/Fraud_caught_per_sample` | 1 when a true fraud was correctly flagged | +| `target`, `prediction` columns | Per-sample truth/pred to sort/lock in List view | + +Class weights (`[1.0, 4.0]`, config) up-weight the fraud class against the ~12% +prevalence so the minority class drives the gradient. + +## Test it + +```bash +# Fast, offline unit tests (pure PyTorch, no gRPC server): +python -m pytest test_fraud_detection.py -v + +# End-to-end integration check (needs weightslab installed): drives the tracked +# loaders + watched loss + gRPC server, then asserts the ledger dataframe the UI +# reads has per-sample rows, every feature as a column, target/prediction/loss, +# and a live gRPC endpoint. +python verify_integration.py +``` + +The unit tests cover the dataset contract, reproducibility, class balance, the +model forward pass, `get_items` metadata columns, and that a few optimizer steps +reduce the loss (final accuracy > 90%). `verify_integration.py` proves WeightsLab +is fully wired in tabular mode. diff --git a/weightslab/examples/PyTorch/wl-fraud-detection/config.yaml b/weightslab/examples/PyTorch/wl-fraud-detection/config.yaml new file mode 100644 index 00000000..01b5a79c --- /dev/null +++ b/weightslab/examples/PyTorch/wl-fraud-detection/config.yaml @@ -0,0 +1,48 @@ +# Global configuration +experiment_name: fraud_detection_mlp +device: auto +training_steps_to_do: null # null = train until manually stopped from the UI +# root_log_dir: ... # Path to save current experiment checkpoints and data + +# Initially compute natural sorting values (e.g. avg feature magnitude). +compute_natural_sort: false + +# Experiment parameters +eval_full_to_train_steps_ratio: 100 +experiment_dump_to_train_steps_ratio: 250 +write_export_ratio: 100 +skip_checkpoint_load: false +tqdm_display: true +is_training: false # Start paused; press play in the UI. + +# Global dataframe storage +ledger_enable_flushing_threads: true +ledger_enable_h5_persistence: true +ledger_flush_max_rows: 15000 +ledger_flush_interval: 30.0 + +# Clients +serving_grpc: true + +# Optimizer +optimizer: + lr: 0.005 + +# Class weighting to counter ~12% fraud prevalence [legit, fraud]. +class_weights: [1.0, 4.0] + +# Synthetic dataset generation (reproducible, no download). +dataset: + seed: 0 + n_train: 4000 + n_test: 1000 + +# DataLoader parameters +data: + train_loader: + shuffle: true + batch_size: 32 + test_loader: + shuffle: false + batch_size: 128 + drop_last: false diff --git a/weightslab/examples/PyTorch/wl-fraud-detection/main.py b/weightslab/examples/PyTorch/wl-fraud-detection/main.py new file mode 100644 index 00000000..4001684b --- /dev/null +++ b/weightslab/examples/PyTorch/wl-fraud-detection/main.py @@ -0,0 +1,284 @@ +"""WeightsLab example: bank card-transaction fraud detection (tabular, PyTorch). + +A small MLP binary classifier over synthetic transaction features, wired into +WeightsLab so the run streams per-sample stats (loss, prediction, target, +discard state) to the UI. Because the data is tabular, this is a natural fit for +the List Exploration (tabular) view — sort by loss or prediction to triage the +transactions the model finds hardest. + +Run: + cd weightslab/examples/PyTorch/wl-fraud-detection + python main.py + +The dataset/model live in ``utils/`` (pure PyTorch) so they can be unit tested +without the gRPC backend — see ``test_fraud_detection.py``. +""" + +import itertools +import os +import time +import logging +import tempfile + +import yaml +import tqdm +import torch +import torch.nn as nn +import torch.optim as optim + +from torchmetrics.classification import Accuracy + +import weightslab as wl +from weightslab.components.global_monitoring import ( + guard_training_context, + guard_testing_context, +) + +from utils.data import FraudDataset, NUM_FEATURES +from utils.model import FraudMLP + + +logging.basicConfig(level=logging.ERROR) +logger = logging.getLogger(__name__) + + +# ----------------------------------------------------------------------------- +# Train / Test steps +# ----------------------------------------------------------------------------- +def train(loader, model, optimizer, criterion_mlt, device): + """Single training step using the tracked dataloader + watched loss.""" + with guard_training_context: + (inputs, ids, labels) = next(loader) + inputs = inputs.to(device) + labels = labels.to(device) + + optimizer.zero_grad() + preds_raw = model(inputs) + preds = preds_raw.argmax(dim=1, keepdim=True) + + loss_batch_mlt = criterion_mlt( + preds_raw.float(), + labels.long(), + batch_ids=ids, + preds=preds, + ) + total_loss = loss_batch_mlt.mean() + + total_loss.backward() + optimizer.step() + + return total_loss.detach().cpu().item() + + +def test(loader, model, criterion_mlt, metric_mlt, device, test_loader_len): + """Full evaluation pass over the test loader, logging per-sample signals.""" + losses = torch.tensor(0.0, device=device) + + for (inputs, ids, labels) in loader: + with guard_testing_context: + inputs = inputs.to(device) + labels = labels.to(device) + + outputs = model(inputs) + preds = outputs.argmax(dim=1, keepdim=True) + + loss_batch = criterion_mlt(outputs, labels, batch_ids=ids, preds=preds) + losses += torch.mean(loss_batch) + metric_mlt.update(outputs, labels) + + preds_flat = preds.view(-1) + labels_flat = labels.view(-1) + acc_per_sample = (preds_flat == labels_flat).float() + fraud_caught_per_sample = ((preds_flat == 1) & (labels_flat == 1)).float() + + signals = { + "test_metric/Accuracy_per_sample": acc_per_sample, + "test_metric/Fraud_caught_per_sample": fraud_caught_per_sample, + } + wl.save_signals( + preds_raw=outputs, + targets=labels, + batch_ids=ids, + signals=signals, + preds=preds, + ) + + loss = losses / max(1, test_loader_len) + metric = metric_mlt.compute() * 100 + + return loss.detach().cpu().item(), metric.detach().cpu().item() + + +# ----------------------------------------------------------------------------- +# Main +# ----------------------------------------------------------------------------- +if __name__ == "__main__": + start_time = time.time() + + # Load hyperparameters (from YAML if present). + parameters = {} + config_path = os.path.join(os.path.dirname(__file__), "config.yaml") + if os.path.exists(config_path): + with open(config_path, "r") as fh: + parameters = yaml.safe_load(fh) or {} + parameters = parameters or {} + + # ---- sensible defaults / normalization ---- + parameters.setdefault("experiment_name", "fraud_detection_mlp") + parameters.setdefault("device", "auto") + parameters.setdefault("training_steps_to_do", 1000000) + parameters.setdefault("eval_full_to_train_steps_ratio", 100) + + exp_name = parameters["experiment_name"] + + # Hyperparameters (must use 'hyperparameters' flag for trainer services / UI). + wl.watch_or_edit(parameters, flag="hyperparameters", poll_interval=1.0) + + # Device selection + if parameters.get("device", "auto") == "auto": + parameters["device"] = torch.device("cuda" if torch.cuda.is_available() else "cpu") + device = parameters["device"] + + # Logging dir + if not parameters.get("root_log_dir"): + parameters["root_log_dir"] = tempfile.mkdtemp() + print(f"No root_log_dir specified, using temporary directory: {parameters['root_log_dir']}") + os.makedirs(parameters["root_log_dir"], exist_ok=True) + + verbose = parameters.get("verbose", True) + log_dir = parameters["root_log_dir"] + tqdm_display = parameters.get("tqdm_display", True) + eval_full_to_train_steps_ratio = parameters.get("eval_full_to_train_steps_ratio", 100) + write_export_ratio = parameters.get("write_export_ratio", 100) + enable_h5_persistence = parameters.get("enable_h5_persistence", True) + training_steps_to_do = parameters.get("training_steps_to_do", 1000) + + # Model + _model = FraudMLP(in_features=NUM_FEATURES, num_classes=2).to(device) + model = wl.watch_or_edit(_model, flag="model", device=device) + + # Optimizer + lr = parameters.get("optimizer", {}).get("lr", 0.005) + _optimizer = optim.Adam(model.parameters(), lr=lr) + optimizer = wl.watch_or_edit(_optimizer, flag="optimizer") + + # Data (synthetic tabular fraud stream) — no download needed. + dataset_cfg = parameters.get("dataset", {}) + seed = int(dataset_cfg.get("seed", 0)) + n_train = int(dataset_cfg.get("n_train", 4000)) + n_test = int(dataset_cfg.get("n_test", 1000)) + + train_cfg = parameters.get("data", {}).get("train_loader", {}) + test_cfg = parameters.get("data", {}).get("test_loader", {}) + + _train_dataset = FraudDataset(n_train, seed=seed, max_samples=train_cfg.get("max_samples")) + _test_dataset = FraudDataset(n_test, seed=seed + 1, max_samples=test_cfg.get("max_samples")) + + train_loader = wl.watch_or_edit( + _train_dataset, + flag="data", + loader_name="train_loader", + batch_size=train_cfg.get("batch_size", 32), + shuffle=train_cfg.get("shuffle", True), + is_training=True, + compute_hash=False, + preload_labels=True, + preload_metadata=True, + enable_h5_persistence=enable_h5_persistence, + ) + test_loader = wl.watch_or_edit( + _test_dataset, + flag="data", + loader_name="test_loader", + batch_size=test_cfg.get("batch_size", 128), + shuffle=test_cfg.get("shuffle", False), + is_training=False, + compute_hash=False, + preload_labels=True, + preload_metadata=True, + enable_h5_persistence=enable_h5_persistence, + ) + + # Losses & metrics (watched objects – they log themselves). + # Class weighting counters the ~12% fraud prevalence so the minority class + # actually drives the gradient. + class_weights = torch.tensor( + parameters.get("class_weights", [1.0, 4.0]), dtype=torch.float32, device=device + ) + train_criterion = wl.watch_or_edit( + nn.CrossEntropyLoss(weight=class_weights, reduction="none"), + flag="loss", signal_name="train-loss-CE", log=True) + test_criterion = wl.watch_or_edit( + nn.CrossEntropyLoss(weight=class_weights, reduction="none"), + flag="loss", signal_name="test-loss-CE", log=True) + + metric = wl.watch_or_edit( + Accuracy(task="multiclass", num_classes=2).to(device), + flag="metric", signal_name="metric-ACC", log=True) + + # Start WeightsLab services (gRPC only, no CLI). + wl.serve(serving_grpc=parameters.get("serving_grpc", False)) + + print("=" * 60) + print(" STARTING FRAUD-DETECTION TRAINING") + print(f" Evaluation every {eval_full_to_train_steps_ratio} steps") + print(f" Dataset splits: train={len(_train_dataset)}, test={len(_test_dataset)}") + print(f" Logs will be saved to: {log_dir}") + print("=" * 60 + "\n") + + if tqdm_display: + train_range = tqdm.tqdm( + range(training_steps_to_do) if training_steps_to_do != None else itertools.count(), + desc="Training", + bar_format="{desc}: {n}/{total} [{elapsed}<{remaining}, {rate_fmt}] {bar} | {postfix}", + ncols=140, + position=0, + leave=True, + ) + else: + train_range = range(training_steps_to_do) if training_steps_to_do != None else itertools.count() + + # ================ + # Training Loop + wl.start_training(timeout=3) + + train_loss = None + test_loss, test_metric = None, None + test_loader_len = len(test_loader) + for train_step in train_range: + age = model.get_age() if hasattr(model, "get_age") else train_step + + train_loss = train(train_loader, model, optimizer, train_criterion, device) + + if age > 0 and age % eval_full_to_train_steps_ratio == 0: + test_loss, test_metric = test( + test_loader, model, test_criterion, metric, device, test_loader_len + ) + + if age > 0 and age % write_export_ratio == 0: + wl.write_history() + wl.write_dataframe() + + if verbose and not tqdm_display: + import sys + msg = f"Step {train_step} (Age {age}): Loss={train_loss:.4f}" + if test_loss is not None: + msg += f" | Test={test_loss:.4f} ({test_metric:.1f}%)" + sys.stdout.write(f"\r{msg:<100}") + sys.stdout.flush() + elif tqdm_display: + postfix_parts = [f"train_loss={train_loss:.4f}"] + if test_loss is not None: + postfix_parts.append(f"test_loss={test_loss:.4f}") + if test_metric is not None: + postfix_parts.append(f"test_acc={test_metric:.1f}%") + train_range.set_postfix_str(" | ".join(postfix_parts)) + + print("\n" + "=" * 60) + print(f" Training completed in {time.time() - start_time:.2f} seconds") + print(f" Logs saved to: {log_dir}") + print("=" * 60) + + wl.write_history() + wl.write_dataframe() + wl.keep_serving() diff --git a/weightslab/examples/PyTorch/wl-fraud-detection/test_fraud_detection.py b/weightslab/examples/PyTorch/wl-fraud-detection/test_fraud_detection.py new file mode 100644 index 00000000..21c4e21e --- /dev/null +++ b/weightslab/examples/PyTorch/wl-fraud-detection/test_fraud_detection.py @@ -0,0 +1,118 @@ +"""Smoke tests for the fraud-detection example (pure PyTorch, no weightslab). + +Run: python -m pytest test_fraud_detection.py + or: python test_fraud_detection.py +""" + +import os +import sys + +import torch + +sys.path.insert(0, os.path.dirname(__file__)) + +from utils.data import ( # noqa: E402 + FEATURE_NAMES, + IMG_SIDE, + NUM_FEATURES, + FraudDataset, + make_synthetic_fraud, +) +from utils.model import FraudMLP # noqa: E402 + + +def test_feature_layout_matches_image_grid(): + assert NUM_FEATURES == len(FEATURE_NAMES) == 16 + assert IMG_SIDE * IMG_SIDE == NUM_FEATURES + + +def test_synthetic_shapes_and_labels(): + x_std, x_raw, y = make_synthetic_fraud(500, seed=0) + assert x_std.shape == x_raw.shape == (500, NUM_FEATURES) + assert y.shape == (500,) + assert str(x_std.dtype) == "float32" + assert set(int(v) for v in set(y.tolist())) <= {0, 1} + # Standardized features are ~zero-mean, raw ones are not. + assert abs(float(x_std.mean())) < 0.1 + assert abs(float(x_raw.mean())) > 0.1 + + +def test_synthetic_is_deterministic_for_a_seed(): + x1, _, y1 = make_synthetic_fraud(200, seed=42) + x2, _, y2 = make_synthetic_fraud(200, seed=42) + assert (x1 == x2).all() and (y1 == y2).all() + x3, _, _ = make_synthetic_fraud(200, seed=7) + assert not (x1 == x3).all() + + +def test_class_imbalance_is_realistic(): + _, _, y = make_synthetic_fraud(1000, seed=1) + assert 0.05 < float(y.mean()) < 0.20 + + +def test_dataset_item_contract(): + ds = FraudDataset(300, seed=3) + assert len(ds) == 300 + x, idx, label = ds[0] + # Model input is the 1-D feature vector (no fake image). + assert tuple(x.shape) == (NUM_FEATURES,) + assert x.dtype == torch.float32 + assert idx == 0 and label in (0, 1) + + +def test_get_items_exposes_feature_metadata_columns(): + """The ledger-init contract must return raw features as a metadata dict so + they become sortable columns in the WeightsLab UI.""" + ds = FraudDataset(50, seed=0) + image, uid, target, metadata = ds.get_items( + 3, include_metadata=True, include_labels=True, include_images=False + ) + assert image is None # init does not decode images + assert uid == 3 and target in (0, 1) + assert isinstance(metadata, dict) + assert set(metadata.keys()) == set(FEATURE_NAMES) + assert all(isinstance(v, float) for v in metadata.values()) + # Raw values, not standardized (e.g. amount is a positive currency figure). + assert metadata["amount"] != 0.0 + + +def test_dataset_respects_max_samples(): + assert len(FraudDataset(500, seed=0, max_samples=64)) == 64 + + +def test_model_forward_shape_flat_and_image(): + model = FraudMLP() + assert model(torch.randn(8, NUM_FEATURES)).shape == (8, 2) + assert model(torch.randn(8, 1, IMG_SIDE, IMG_SIDE)).shape == (8, 2) + + +def test_training_reduces_loss(): + torch.manual_seed(0) + ds = FraudDataset(1000, seed=0) + features = ds.features # [N, NUM_FEATURES] + labels = ds.labels + + model = FraudMLP() + optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) + criterion = torch.nn.CrossEntropyLoss() + + model.train() + initial_loss = criterion(model(features), labels).item() + for _ in range(50): + optimizer.zero_grad() + loss = criterion(model(features), labels) + loss.backward() + optimizer.step() + final_loss = criterion(model(features), labels).item() + assert final_loss < initial_loss * 0.85, f"{initial_loss:.4f} -> {final_loss:.4f}" + + model.eval() + with torch.no_grad(): + acc = float((model(features).argmax(dim=1) == labels).float().mean()) + assert acc > 0.9, f"accuracy too low: {acc:.3f}" + + +if __name__ == "__main__": + import pytest + + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/weightslab/examples/PyTorch/wl-fraud-detection/utils/__init__.py b/weightslab/examples/PyTorch/wl-fraud-detection/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/weightslab/examples/PyTorch/wl-fraud-detection/utils/data.py b/weightslab/examples/PyTorch/wl-fraud-detection/utils/data.py new file mode 100644 index 00000000..d0d62873 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-fraud-detection/utils/data.py @@ -0,0 +1,189 @@ +"""Synthetic bank card-transaction fraud dataset (pure PyTorch). + +Import-light (only ``numpy`` + ``torch``) so it can be unit-tested without +importing ``weightslab`` or starting the gRPC backend — see +``test_fraud_detection.py``. + +This is a reproducible, offline stand-in for a real card-fraud stream. Each row +is a transaction described by 16 numeric features, with a binary label +(0 = legitimate, 1 = fraud, ~12% prevalence). + +Real-world analogues (drop-in replaceable — just swap the dataset): + * Kaggle "Credit Card Fraud Detection" (ULB, ``creditcard.csv``, 284k txns, + anonymized V1..V28 PCA features) — https://www.kaggle.com/mlg-ulb/creditcardfraud + * PaySim mobile-money fraud simulator — https://www.kaggle.com/ealaxi/paysim1 + +The model input is the 1-D feature vector itself (no fake image). WeightsLab +transmits it through gRPC as a ``vector`` raw_data stat carrying the actual +values, and ``get_items`` exposes the raw features as sortable metadata columns +in the List Exploration (tabular) view. +""" + +from __future__ import annotations + +from typing import Optional, Tuple + +import numpy as np +import torch +from torch.utils.data import Dataset + +# 16 features -> a tidy 4x4 grid, no padding needed. +FEATURE_NAMES = [ + "amount", + "old_balance", + "new_balance", + "balance_delta", + "txn_hour", + "txn_day_of_week", + "txn_count_1h", + "txn_count_24h", + "avg_amount_7d", + "std_amount_7d", + "merchant_risk", + "device_change", + "geo_distance_km", + "is_foreign", + "account_age_days", + "num_prior_disputes", +] +NUM_FEATURES = len(FEATURE_NAMES) # 16 +IMG_SIDE = 4 +assert IMG_SIDE * IMG_SIDE == NUM_FEATURES + +FRAUD_RATE = 0.12 + + +def make_synthetic_fraud( + n_samples: int, seed: int = 0 +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Generate deterministic synthetic transactions. + + Returns ``(X_std, X_raw, y)``: + * ``X_std``: ``float32[n, NUM_FEATURES]`` standardized features (model input) + * ``X_raw``: ``float32[n, NUM_FEATURES]`` raw, human-readable feature values + (surfaced as sortable metadata columns in the UI) + * ``y``: ``int64[n]`` label, 1 = fraud + + Fraud rows are drawn from shifted distributions (larger amounts and balance + deltas, off-hours activity, higher merchant risk, more device changes, larger + geo jumps, more prior disputes). The signal is separable enough that a small + MLP learns it quickly, while class overlap keeps it non-trivial. + """ + if n_samples <= 0: + empty = np.zeros((0, NUM_FEATURES), dtype=np.float32) + return (empty, empty.copy(), np.zeros((0,), dtype=np.int64)) + + rng = np.random.default_rng(seed) + n_fraud = max(1, int(round(n_samples * FRAUD_RATE))) + n_legit = max(1, n_samples - n_fraud) + + def _draw(n: int, fraud: bool) -> list[dict]: + """One dict per row, keyed by FEATURE_NAMES — self-documenting, and the + final array assembly below reads the keys in FEATURE_NAMES order, so a + renamed/reordered feature can't silently desync from its column.""" + amount = rng.gamma(2.0, 180.0 if fraud else 60.0, size=n) + old_balance = rng.gamma(2.0, 800.0, size=n) + spent = amount * (rng.uniform(0.6, 1.4, n) if fraud else rng.uniform(0.0, 0.6, n)) + new_balance = np.clip(old_balance - spent, 0, None) + balance_delta = old_balance - new_balance + txn_hour = (rng.normal(2.5, 2.0, n) % 24) if fraud else rng.normal(13.0, 4.0, n) + txn_day_of_week = rng.integers(0, 7, n).astype(np.float64) + txn_count_1h = rng.poisson(4.0 if fraud else 1.0, n).astype(np.float64) + txn_count_24h = rng.poisson(18.0 if fraud else 6.0, n).astype(np.float64) + avg_amount_7d = rng.gamma(2.0, 90.0 if fraud else 70.0, size=n) + std_amount_7d = rng.gamma(2.0, 60.0 if fraud else 25.0, size=n) + merchant_risk = rng.beta(5.0, 2.0, n) if fraud else rng.beta(2.0, 6.0, n) + device_change = rng.binomial(1, 0.55 if fraud else 0.08, n).astype(np.float64) + geo_distance_km = rng.gamma(2.0, 400.0 if fraud else 25.0, size=n) + is_foreign = rng.binomial(1, 0.45 if fraud else 0.05, n).astype(np.float64) + account_age_days = rng.gamma(2.0, 120.0 if fraud else 500.0, size=n) + num_prior_disputes = rng.poisson(1.5 if fraud else 0.2, n).astype(np.float64) + + return [ + {'amount': amount[i], + 'old_balance': old_balance[i], + 'new_balance': new_balance[i], + 'balance_delta': balance_delta[i], + 'txn_hour': txn_hour[i], + 'txn_day_of_week': txn_day_of_week[i], + 'txn_count_1h': txn_count_1h[i], + 'txn_count_24h': txn_count_24h[i], + 'avg_amount_7d': avg_amount_7d[i], + 'std_amount_7d': std_amount_7d[i], + 'merchant_risk': merchant_risk[i], + 'device_change': device_change[i], + 'geo_distance_km': geo_distance_km[i], + 'is_foreign': is_foreign[i], + 'account_age_days': account_age_days[i], + 'num_prior_disputes': num_prior_disputes[i]} for i in range(n) + ] + + rows = _draw(n_legit, False) + _draw(n_fraud, True) + x_raw = np.array([[row[name] for name in FEATURE_NAMES] for row in rows], dtype=np.float64) + y = np.concatenate([np.zeros(n_legit, dtype=np.int64), np.ones(n_fraud, dtype=np.int64)]) + + # Standardize per-feature (class-agnostic) so the MLP trains stably. + mean = x_raw.mean(axis=0, keepdims=True) + std = x_raw.std(axis=0, keepdims=True) + std[std == 0] = 1.0 + x_std = (x_raw - mean) / std + + perm = rng.permutation(len(x_raw)) # interleave fraud/legit deterministically + return ( + x_std[perm].astype(np.float32), + x_raw[perm].astype(np.float32), + y[perm], + ) + + +class FraudDataset(Dataset): + """Tabular fraud dataset yielding ``(input, idx, label)``. + + ``input`` is the 1-D standardized feature vector ``float32[NUM_FEATURES]`` + fed straight to the model — there is no image. WeightsLab transmits the + feature values through gRPC as a ``vector`` raw_data stat, and ``get_items`` + exposes the raw values as sortable metadata columns. ``idx`` is the tracked + sample id; ``label`` is 0 (legit) / 1 (fraud). + """ + + def __init__(self, n_samples: int, seed: int = 0, max_samples: Optional[int] = None): + x_std, x_raw, y = make_synthetic_fraud(n_samples, seed=seed) + if max_samples is not None: + x_std, x_raw, y = x_std[:max_samples], x_raw[:max_samples], y[:max_samples] + self.features = torch.from_numpy(x_std) # [N, 16] float32 (model input) + self.raw = x_raw # [N, 16] float32 (display values) + self.labels = torch.from_numpy(y) # [N] int64 + + def __len__(self) -> int: + return int(self.features.shape[0]) + + def _input(self, idx: int) -> torch.Tensor: + # Tabular: the model input IS the 1-D feature vector (no fake image). + return self.features[idx] + + def _metadata(self, idx: int) -> dict: + """Raw, human-readable feature values -> sortable UI columns.""" + row = self.raw[idx] + return {name: round(float(row[i]), 4) for i, name in enumerate(FEATURE_NAMES)} + + def __getitems__(self, idx: int): + # Training contract: (input, sample_id, label). + return self._input(idx), idx, int(self.labels[idx].item()) + + def __getitem__(self, idx: int): + # Training contract: (input, sample_id, label). + return self._input(idx), idx, int(self.labels[idx].item()) + + def get_items(self, idx: int, include_metadata: bool = False, + include_labels: bool = False, include_images: bool = False): + """WeightsLab ledger-init contract: (image, uid, target, metadata). + + Called once per sample at init with ``include_images=False``; the + returned ``metadata`` dict is flattened into per-sample columns, so each + transaction feature (``amount``, ``merchant_risk``, …) becomes a + sortable column in the List Exploration view. + """ + image = self._input(idx) if include_images else None + target = int(self.labels[idx].item()) if include_labels else None + metadata = self._metadata(idx) if include_metadata else None + return image, idx, target, metadata diff --git a/weightslab/examples/PyTorch/wl-fraud-detection/utils/model.py b/weightslab/examples/PyTorch/wl-fraud-detection/utils/model.py new file mode 100644 index 00000000..61503096 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-fraud-detection/utils/model.py @@ -0,0 +1,33 @@ +"""Small MLP binary classifier for tabular fraud detection (pure PyTorch).""" + +from __future__ import annotations + +import torch +import torch.nn as nn + +from .data import NUM_FEATURES, IMG_SIDE + + +class FraudMLP(nn.Module): + """MLP over the 16 transaction features. + + Accepts either flat ``[N, 16]`` or image-shaped ``[N, 1, 4, 4]`` input and + outputs 2 logits (legit / fraud), so it plugs into ``CrossEntropyLoss`` and a + 2-class accuracy metric like the other WeightsLab usecases. + """ + + def __init__(self, in_features: int = NUM_FEATURES, hidden: int = 64, num_classes: int = 2): + super().__init__() + self.input_shape = (1, NUM_FEATURES) + self.net = nn.Sequential( + nn.Flatten(), + nn.Linear(in_features, hidden), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden, hidden // 2), + nn.ReLU(), + nn.Linear(hidden // 2, num_classes), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.net(x) diff --git a/weightslab/examples/PyTorch/wl-fraud-detection/verify_integration.py b/weightslab/examples/PyTorch/wl-fraud-detection/verify_integration.py new file mode 100644 index 00000000..93de776f --- /dev/null +++ b/weightslab/examples/PyTorch/wl-fraud-detection/verify_integration.py @@ -0,0 +1,221 @@ +"""Integration check: prove WeightsLab is fully wired for this tabular example. + +Unlike ``test_fraud_detection.py`` (pure PyTorch), this actually drives the +WeightsLab stack end-to-end — tracked dataloaders, watched loss/metric, the +gRPC server, and the per-sample ledger — then asserts the sample dataframe the +UI reads contains, for tabular mode: + + * one row per sample (what you scroll/sort in the grid & List view), + * every transaction feature as its own column (from ``get_items`` metadata), + * ``target`` + ``prediction`` populated (so sort/filter/histograms work), + * a per-sample training/eval loss column, + +and that the gRPC server is actually listening (so Weights Studio can connect +live during the run). + +Run: python verify_integration.py +Requires weightslab installed (not a pure-unit test). +""" + +import glob +import os +import socket +import sys +import tempfile + +import torch +import torch.nn as nn +import torch.optim as optim +from torchmetrics.classification import Accuracy + +sys.path.insert(0, os.path.dirname(__file__)) + +import weightslab as wl # noqa: E402 +from weightslab.components.global_monitoring import ( # noqa: E402 + guard_training_context, + guard_testing_context, +) + +from utils.data import FraudDataset, FEATURE_NAMES, NUM_FEATURES # noqa: E402 +from utils.model import FraudMLP # noqa: E402 + + +def _free_port() -> int: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +def main() -> int: + device = torch.device("cpu") + log_dir = tempfile.mkdtemp(prefix="wl_fraud_verify_") + grpc_port = _free_port() + # The gRPC server reads its port from GRPC_BACKEND_PORT (same knob the E2E + # harness uses); set it before serve() so we bind to a free ephemeral port. + os.environ["GRPC_BACKEND_PORT"] = str(grpc_port) + + parameters = { + "experiment_name": "fraud_verify", + "device": "cpu", + "root_log_dir": log_dir, + "serving_grpc": True, + "grpc_port": grpc_port, + "is_training": True, + } + wl.watch_or_edit(parameters, flag="hyperparameters", poll_interval=1.0) + + model = wl.watch_or_edit(FraudMLP(in_features=NUM_FEATURES, num_classes=2).to(device), + flag="model", device=device) + optimizer = wl.watch_or_edit(optim.Adam(model.parameters(), lr=0.005), flag="optimizer") + + train_ds = FraudDataset(600, seed=0) + test_ds = FraudDataset(200, seed=1) + + train_loader = wl.watch_or_edit( + train_ds, flag="data", loader_name="train_loader", batch_size=32, shuffle=True, + is_training=True, compute_hash=False, preload_labels=True, preload_metadata=True, + enable_h5_persistence=False) + test_loader = wl.watch_or_edit( + test_ds, flag="data", loader_name="test_loader", batch_size=64, shuffle=False, + is_training=False, compute_hash=False, preload_labels=True, preload_metadata=True, + enable_h5_persistence=False) + + cw = torch.tensor([1.0, 4.0]) + train_crit = wl.watch_or_edit(nn.CrossEntropyLoss(weight=cw, reduction="none"), + flag="loss", signal_name="train-loss-CE", log=True) + test_crit = wl.watch_or_edit(nn.CrossEntropyLoss(weight=cw, reduction="none"), + flag="loss", signal_name="test-loss-CE", log=True) + metric = wl.watch_or_edit(Accuracy(task="multiclass", num_classes=2), + flag="metric", signal_name="metric-ACC", log=True) + + wl.serve(serving_grpc=True, grpc_port=grpc_port) + wl.start_training() + + # ---- drive a few real training steps ---- + for _ in range(120): + with guard_training_context: + inputs, ids, labels = next(train_loader) + optimizer.zero_grad() + out = model(inputs) + preds = out.argmax(dim=1, keepdim=True) + loss = train_crit(out, labels, batch_ids=ids, preds=preds).mean() + loss.backward() + optimizer.step() + + # ---- one full eval pass (populates test loss/prediction per sample) ---- + for inputs, ids, labels in test_loader: + with guard_testing_context: + out = model(inputs) + preds = out.argmax(dim=1, keepdim=True) + test_crit(out, labels, batch_ids=ids, preds=preds) + metric.update(out, labels) + + wl.drain_signals() + + # ---- dump the ledger dataframe the UI reads and inspect it ---- + wl.write_dataframe(path=log_dir, format="csv") + csvs = glob.glob(os.path.join(log_dir, "*dataframe*.csv")) + assert csvs, f"no dataframe CSV written to {log_dir}" + with open(csvs[0], "r", encoding="utf-8") as fh: + header = fh.readline().strip().split(",") + rows = [ln for ln in fh.read().splitlines() if ln.strip()] + + cols = set(c.strip().strip('"') for c in header) + n_rows = len(rows) + + print("\n=== Ledger dataframe ===") + print(f"rows: {n_rows}") + print(f"columns ({len(cols)}): {sorted(cols)}") + + # ---- assertions ---- + problems = [] + if n_rows < 700: # 600 train + 200 test - some may be batched off; expect most + problems.append(f"expected ~800 sample rows, got {n_rows}") + + missing_features = [f for f in FEATURE_NAMES if f not in cols] + if missing_features: + problems.append(f"missing feature columns: {missing_features}") + + for required in ("target", "prediction"): + if required not in cols: + problems.append(f"missing '{required}' column") + + if not any("loss" in c.lower() for c in cols): + problems.append("no per-sample loss column found") + + # gRPC server must be reachable so Weights Studio can connect live. + # It starts on a background thread, so poll for a few seconds. + import time as _time + grpc_up = False + for _ in range(20): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(1) + if sock.connect_ex(("127.0.0.1", grpc_port)) == 0: + sock.close() + grpc_up = True + break + sock.close() + _time.sleep(0.5) + if not grpc_up: + problems.append(f"gRPC server not listening on {grpc_port}") + else: + print(f"gRPC server listening on 127.0.0.1:{grpc_port} ✓") + + # Real gRPC round-trip: the input the model was fed (the feature vector) + # must reach the UI as raw_data of type 'vector' carrying the values. + try: + import grpc + from weightslab.proto import experiment_service_pb2 as pb2 + from weightslab.proto import experiment_service_pb2_grpc as pb2_grpc + + channel = grpc.insecure_channel(f"127.0.0.1:{grpc_port}") + grpc.channel_ready_future(channel).result(timeout=10) + stub = pb2_grpc.ExperimentServiceStub(channel) + resp = stub.GetDataSamples(pb2.DataSamplesRequest( + start_index=0, records_cnt=5, include_raw_data=True, + resize_width=0, resize_height=0), timeout=20) + + records = list(resp.data_records) + raw = None + for rec in records: + for st in rec.data_stats: + if st.name == "raw_data": + raw = st + break + if raw is not None: + break + + if raw is None: + problems.append("gRPC GetDataSamples returned no raw_data stat") + elif raw.type != "vector": + problems.append(f"raw_data.type is '{raw.type}', expected 'vector'") + elif len(raw.value) != NUM_FEATURES: + problems.append( + f"raw_data carries {len(raw.value)} values, expected {NUM_FEATURES}") + else: + print(f"gRPC GetDataSamples: raw_data type='vector', " + f"{len(raw.value)} feature values reached the UI ✓") + print(f" sample values: {[round(v, 3) for v in raw.value[:6]]} …") + channel.close() + except Exception as e: + problems.append(f"gRPC GetDataSamples failed: {e}") + + if problems: + print("\n VERIFY FAILED:") + for p in problems: + print(f" - {p}") + return 1 + + print("\n VERIFY PASSED: WeightsLab is fully wired for tabular mode " + "(per-sample rows + feature columns + target/prediction/loss + live gRPC).") + return 0 + + +if __name__ == "__main__": + code = main() + sys.stdout.flush() + sys.stderr.flush() + # Hard-exit so background serving threads don't keep the process alive. + os._exit(code) diff --git a/weightslab/examples/PyTorch/ws-generation/config.yaml b/weightslab/examples/PyTorch/wl-generation/config.yaml similarity index 100% rename from weightslab/examples/PyTorch/ws-generation/config.yaml rename to weightslab/examples/PyTorch/wl-generation/config.yaml diff --git a/weightslab/examples/PyTorch/ws-generation/main.py b/weightslab/examples/PyTorch/wl-generation/main.py similarity index 100% rename from weightslab/examples/PyTorch/ws-generation/main.py rename to weightslab/examples/PyTorch/wl-generation/main.py diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/0000f77c-6257be58.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/0000f77c-6257be58.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/0000f77c-6257be58.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/0000f77c-6257be58.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/000f8d37-d4c09a0f.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/000f8d37-d4c09a0f.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/000f8d37-d4c09a0f.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/000f8d37-d4c09a0f.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00a0f008-a315437f.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00a0f008-a315437f.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00a0f008-a315437f.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00a0f008-a315437f.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00c12bd0-bb46e479.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00c12bd0-bb46e479.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00c12bd0-bb46e479.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00c12bd0-bb46e479.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00c29c52-f9524f1e.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00c29c52-f9524f1e.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00c29c52-f9524f1e.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00c29c52-f9524f1e.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00ce6f6d-50bbee62.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00ce6f6d-50bbee62.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00ce6f6d-50bbee62.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00ce6f6d-50bbee62.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00d1bafa-1b47b41c.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00d1bafa-1b47b41c.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00d1bafa-1b47b41c.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00d1bafa-1b47b41c.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00d7268f-fd4487be.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00d7268f-fd4487be.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00d7268f-fd4487be.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00d7268f-fd4487be.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00f0dd0f-5e9c9557.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00f0dd0f-5e9c9557.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/00f0dd0f-5e9c9557.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/00f0dd0f-5e9c9557.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/0a0d7f4c-ac5c3c8f.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/0a0d7f4c-ac5c3c8f.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/0a0d7f4c-ac5c3c8f.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/0a0d7f4c-ac5c3c8f.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/0a1f4fce-f9cac880.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/0a1f4fce-f9cac880.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/0a1f4fce-f9cac880.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/0a1f4fce-f9cac880.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/0a3bb2d8-c195d91e.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/0a3bb2d8-c195d91e.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/train/0a3bb2d8-c195d91e.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/train/0a3bb2d8-c195d91e.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1cac6a7-04e33135.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1cac6a7-04e33135.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1cac6a7-04e33135.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1cac6a7-04e33135.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1ceb32e-3f481b43.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1ceb32e-3f481b43.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1ceb32e-3f481b43.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1ceb32e-3f481b43.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1d10d08-5b108225.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1d10d08-5b108225.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1d10d08-5b108225.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1d10d08-5b108225.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1d22449-15fb948f.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1d22449-15fb948f.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1d22449-15fb948f.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1d22449-15fb948f.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1d7b3ac-5af8623b.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1d7b3ac-5af8623b.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1d7b3ac-5af8623b.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1d7b3ac-5af8623b.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1dce572-c6a8cb5e.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1dce572-c6a8cb5e.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1dce572-c6a8cb5e.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1dce572-c6a8cb5e.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1e1a7b8-0aec80e8.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1e1a7b8-0aec80e8.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1e1a7b8-0aec80e8.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1e1a7b8-0aec80e8.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1e8ad72-c3c79240.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1e8ad72-c3c79240.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1e8ad72-c3c79240.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1e8ad72-c3c79240.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1ee702d-0ae1fc10.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1ee702d-0ae1fc10.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1ee702d-0ae1fc10.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1ee702d-0ae1fc10.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1f0efd9-37a14dda.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1f0efd9-37a14dda.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b1f0efd9-37a14dda.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b1f0efd9-37a14dda.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b2a0648b-d8e126bc.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b2a0648b-d8e126bc.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b2a0648b-d8e126bc.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b2a0648b-d8e126bc.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b2b70230-bad4ff6e.jpg b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b2b70230-bad4ff6e.jpg similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/images/val/b2b70230-bad4ff6e.jpg rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/images/val/b2b70230-bad4ff6e.jpg diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/0000f77c-6257be58.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/0000f77c-6257be58.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/0000f77c-6257be58.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/0000f77c-6257be58.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/000f8d37-d4c09a0f.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/000f8d37-d4c09a0f.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/000f8d37-d4c09a0f.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/000f8d37-d4c09a0f.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00a0f008-a315437f.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00a0f008-a315437f.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00a0f008-a315437f.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00a0f008-a315437f.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00c12bd0-bb46e479.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00c12bd0-bb46e479.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00c12bd0-bb46e479.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00c12bd0-bb46e479.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00c29c52-f9524f1e.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00c29c52-f9524f1e.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00c29c52-f9524f1e.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00c29c52-f9524f1e.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00ce6f6d-50bbee62.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00ce6f6d-50bbee62.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00ce6f6d-50bbee62.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00ce6f6d-50bbee62.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00d1bafa-1b47b41c.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00d1bafa-1b47b41c.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00d1bafa-1b47b41c.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00d1bafa-1b47b41c.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00d7268f-fd4487be.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00d7268f-fd4487be.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00d7268f-fd4487be.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00d7268f-fd4487be.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00f0dd0f-5e9c9557.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00f0dd0f-5e9c9557.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/00f0dd0f-5e9c9557.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/00f0dd0f-5e9c9557.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/0a0d7f4c-ac5c3c8f.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/0a0d7f4c-ac5c3c8f.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/0a0d7f4c-ac5c3c8f.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/0a0d7f4c-ac5c3c8f.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/0a1f4fce-f9cac880.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/0a1f4fce-f9cac880.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/0a1f4fce-f9cac880.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/0a1f4fce-f9cac880.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/0a3bb2d8-c195d91e.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/0a3bb2d8-c195d91e.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/train/0a3bb2d8-c195d91e.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/train/0a3bb2d8-c195d91e.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1cac6a7-04e33135.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1cac6a7-04e33135.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1cac6a7-04e33135.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1cac6a7-04e33135.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1ceb32e-3f481b43.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1ceb32e-3f481b43.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1ceb32e-3f481b43.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1ceb32e-3f481b43.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1d10d08-5b108225.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1d10d08-5b108225.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1d10d08-5b108225.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1d10d08-5b108225.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1d22449-15fb948f.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1d22449-15fb948f.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1d22449-15fb948f.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1d22449-15fb948f.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1d7b3ac-5af8623b.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1d7b3ac-5af8623b.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1d7b3ac-5af8623b.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1d7b3ac-5af8623b.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1dce572-c6a8cb5e.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1dce572-c6a8cb5e.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1dce572-c6a8cb5e.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1dce572-c6a8cb5e.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1e1a7b8-0aec80e8.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1e1a7b8-0aec80e8.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1e1a7b8-0aec80e8.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1e1a7b8-0aec80e8.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1e8ad72-c3c79240.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1e8ad72-c3c79240.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1e8ad72-c3c79240.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1e8ad72-c3c79240.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1ee702d-0ae1fc10.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1ee702d-0ae1fc10.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1ee702d-0ae1fc10.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1ee702d-0ae1fc10.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1f0efd9-37a14dda.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1f0efd9-37a14dda.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b1f0efd9-37a14dda.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b1f0efd9-37a14dda.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b2a0648b-d8e126bc.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b2a0648b-d8e126bc.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b2a0648b-d8e126bc.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b2a0648b-d8e126bc.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b2b70230-bad4ff6e.png b/weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b2b70230-bad4ff6e.png similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/BDD_subset/labels/val/b2b70230-bad4ff6e.png rename to weightslab/examples/PyTorch/wl-segmentation/BDD_subset/labels/val/b2b70230-bad4ff6e.png diff --git a/weightslab/examples/PyTorch/ws-segmentation/config.yaml b/weightslab/examples/PyTorch/wl-segmentation/config.yaml similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/config.yaml rename to weightslab/examples/PyTorch/wl-segmentation/config.yaml diff --git a/weightslab/examples/PyTorch/ws-segmentation/main.py b/weightslab/examples/PyTorch/wl-segmentation/main.py similarity index 98% rename from weightslab/examples/PyTorch/ws-segmentation/main.py rename to weightslab/examples/PyTorch/wl-segmentation/main.py index f56b75e4..1f7b1b9d 100644 --- a/weightslab/examples/PyTorch/ws-segmentation/main.py +++ b/weightslab/examples/PyTorch/wl-segmentation/main.py @@ -34,14 +34,6 @@ # Train / Test loops (segmentation, using watcher-wrapped loaders) # ============================================================================= -def _instance_batch_idx(labels): - """Flat instance→sample map (sample-major) matching the PerInstance* ordering.""" - return torch.tensor( - [s for s, insts in enumerate(labels) for _ in insts], - dtype=torch.long, - ) - - def _run_instance_signals(sig, outputs, labels, ids, preds, return_metric=False): """Compute + log/save the per-sample AND per-instance Dice (metric) and BCE (loss).""" bce_sample = sig["bce_sample"](outputs, labels, batch_ids=ids, preds=preds) diff --git a/weightslab/examples/PyTorch/ws-segmentation/utils/criterions.py b/weightslab/examples/PyTorch/wl-segmentation/utils/criterions.py similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/utils/criterions.py rename to weightslab/examples/PyTorch/wl-segmentation/utils/criterions.py diff --git a/weightslab/examples/PyTorch/ws-segmentation/utils/data.py b/weightslab/examples/PyTorch/wl-segmentation/utils/data.py similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/utils/data.py rename to weightslab/examples/PyTorch/wl-segmentation/utils/data.py diff --git a/weightslab/examples/PyTorch/ws-segmentation/utils/model.py b/weightslab/examples/PyTorch/wl-segmentation/utils/model.py similarity index 100% rename from weightslab/examples/PyTorch/ws-segmentation/utils/model.py rename to weightslab/examples/PyTorch/wl-segmentation/utils/model.py diff --git a/weightslab/examples/PyTorch/ws-detection/README.md b/weightslab/examples/PyTorch/ws-detection/README.md deleted file mode 100644 index ba8bddc6..00000000 --- a/weightslab/examples/PyTorch/ws-detection/README.md +++ /dev/null @@ -1,102 +0,0 @@ -# WeightsLab — Object Detection (pure PyTorch) - -A small, fully-runnable **object detection** example wired into WeightsLab. It -trains a compact single-shot detector on the **Penn-Fudan Pedestrian** dataset -(~170 real photos, one class: `person`) and streams per-sample / per-instance -losses, IoU, and predicted bounding boxes to the WeightsLab UI. - -Everything here is plain PyTorch + torchvision — no detection framework -(no Ultralytics/Detectron). The only pretrained piece is an ImageNet backbone. - -## Quick start - -From a WeightsLab install, the one-liner (installs this example's -`requirements.txt`, then trains + serves until `Ctrl+C`): - -```bash -weightslab start example --det -``` - -Or run it directly: - -```bash -cd weightslab/examples/PyTorch/ws-detection -pip install -r requirements.txt -python main.py -``` - -The **first run downloads** the Penn-Fudan dataset (~50 MB, into `./data/`) and -the MobileNetV3-Small ImageNet weights (~10 MB, cached by torch). Then open the -UI (e.g. `http://localhost:5173`) to watch training. - -## What you'll see in the UI - -| Signal | Meaning | -| ----------------------- | ---------------------------------------------------- | -| `train_loss/sample` | Per-image training loss (the value being optimized) | -| `test_loss/sample` | Per-image validation loss | -| `train_iou/sample` | Mean IoU per training image | -| `test_iou/sample` | Mean IoU per validation image | -| `train_iou/instance` | IoU per **ground-truth box** `(sample_id, annotation_id)` | -| `test_iou/instance` | Same, on validation | - -Ground-truth and predicted **bounding boxes** are rendered as overlays on each -sample (the dataset and model declare `task_type = "detection"`). - -## How it works - -``` -utils/data.py PennFudanDetectionDataset — downloads Penn-Fudan, derives one - bbox per pedestrian from the instance masks, returns the WL - detection target [N, 6] = [x1, y1, x2, y2, class_id, conf] - normalized to [0, 1]. ImageNet-normalized model inputs. - `det_collate` keeps the variable box count as a per-sample list. - -utils/model.py SmallDetector — ImageNet-pretrained MobileNetV3-Small backbone - (frozen by default) + a small head that predicts ONE box per - cell on an S x S grid: (objectness, tx, ty, tw, th, class...). - `decode_grid` turns raw logits into xyxy boxes. - -utils/criterions.py PerSampleDetectionLoss — YOLO-style objectness + coordinate + - class loss, one differentiable scalar per sample (what WL - backprops). PerSampleIoU / PerInstanceIoU — IoU metrics. - decode_predictions — top-confidence boxes for the UI overlay. - -main.py Wires it all to WeightsLab: watch_or_edit(...) for the logger, - hyperparameters, data loaders, model, optimizer and the - loss/metric signals; serve(); start_training(); train/test loop. -``` - -The detector is genuinely learnable: on a small subset, mean IoU rises from -~0.39 to ~0.83 within ~60 steps. - -## Configuration (`config.yaml`) - -| Key | Default | Notes | -| ---------------------- | ------- | ----------------------------------------------------------- | -| `num_classes` | `1` | Penn-Fudan has one class (`person`). | -| `image_size` | `256` | Square model input (UI shows the original image). | -| `grid_size` | `8` | Detector predicts on an `8 x 8` cell grid. | -| `conf_thresh` | `0.3` | `objectness * class` threshold for displayed predictions. | -| `pretrained_backbone` | `true` | Load ImageNet weights for the MobileNetV3 backbone. | -| `freeze_backbone` | `true` | Train only the head (fast, less data-hungry). Set `false` to fine-tune the whole backbone once the head has warmed up. | -| `data.*.batch_size` | `8` | Per-loader batch size. | -| `data.*.max_samples` | `null` | Cap a split for quick runs (`null` = full split). | - -## Using your own dataset (e.g. traffic lights) - -The model, loss, metrics, `main.py`, and UI rendering are **dataset-agnostic** — -only `utils/data.py` and a couple of config values change: - -1. Write a `Dataset` whose `get_items(idx, ...)` returns - `(image_tensor, uid, target, metadata)`, where `target` is an - `[N, 6]` float array `[x1, y1, x2, y2, class_id, confidence]` **normalized to - `[0, 1]`** (ground-truth confidence = `1.0`). Set `self.task_type = "detection"`, - `self.num_classes`, `self.class_names`, and expose `self.images` (a list of - image paths) so the UI can show the raw image. -2. Reuse `det_collate` unchanged. -3. In `config.yaml`, set `num_classes` to your class count (e.g. `3` for - `red / yellow / green`) and update `class_names` in the dataset / model. - -That's it — multi-class works out of the box (the classification head is already -in the grid prediction; it's just trivial when `num_classes == 1`). diff --git a/weightslab/examples/Ultralytics/ws-detection/config.yaml b/weightslab/examples/Ultralytics/wl-detection/config.yaml similarity index 90% rename from weightslab/examples/Ultralytics/ws-detection/config.yaml rename to weightslab/examples/Ultralytics/wl-detection/config.yaml index 50a4033e..512f60a4 100644 --- a/weightslab/examples/Ultralytics/ws-detection/config.yaml +++ b/weightslab/examples/Ultralytics/wl-detection/config.yaml @@ -38,7 +38,7 @@ ledger_flush_interval: 60.0 # Data num_classes: 2 image_size: 320 -data_root: C:\Users\GuillaumePELLUET\Documents\Codes\weightslab_kitchen\guillaume_playground\ws-ultralytics_yolo\data\data.yaml # Uncomment and set the path to your data.yaml file. YOLO format. +data_root: .\data\data.yaml # Uncomment and set the path to your data.yaml file. YOLO format. data: train_loader: batch_size: 4 diff --git a/weightslab/examples/Ultralytics/ws-detection/main.py b/weightslab/examples/Ultralytics/wl-detection/main.py similarity index 100% rename from weightslab/examples/Ultralytics/ws-detection/main.py rename to weightslab/examples/Ultralytics/wl-detection/main.py diff --git a/weightslab/examples/Usecases/ws-2d-lidar-detection/README.md b/weightslab/examples/Usecases/wl-2d-lidar-detection/README.md similarity index 94% rename from weightslab/examples/Usecases/ws-2d-lidar-detection/README.md rename to weightslab/examples/Usecases/wl-2d-lidar-detection/README.md index 1925b9f4..46174c1a 100644 --- a/weightslab/examples/Usecases/ws-2d-lidar-detection/README.md +++ b/weightslab/examples/Usecases/wl-2d-lidar-detection/README.md @@ -1,12 +1,12 @@ # 2D LiDAR (laser-scan) Object Detection (Pillars2D-lite) -The 2D sibling of [`ws-3d-lidar-detection`](../ws-3d-lidar-detection/): object +The 2D sibling of [`wl-3d-lidar-detection`](../wl-3d-lidar-detection/): object detection on a **2D point cloud** (a single-layer laser scan / bird's-eye occupancy slice) instead of a 3D LiDAR scene. Same WeightsLab wiring and per-sample / per-instance signals — just with `z` and `yaw` dropped. ``` -ws-2d-lidar-detection/ +wl-2d-lidar-detection/ main.py # WL wiring + train/eval loop config.yaml # hyperparameters, plane range, loaders utils/ @@ -18,7 +18,7 @@ ws-2d-lidar-detection/ ## Quick start ```bash -cd weightslab/examples/Usecases/ws-2d-lidar-detection +cd weightslab/examples/Usecases/wl-2d-lidar-detection python main.py # or, from anywhere: weightslab start example --2d_det diff --git a/weightslab/examples/Usecases/ws-2d-lidar-detection/config.yaml b/weightslab/examples/Usecases/wl-2d-lidar-detection/config.yaml similarity index 100% rename from weightslab/examples/Usecases/ws-2d-lidar-detection/config.yaml rename to weightslab/examples/Usecases/wl-2d-lidar-detection/config.yaml diff --git a/weightslab/examples/Usecases/ws-2d-lidar-detection/main.py b/weightslab/examples/Usecases/wl-2d-lidar-detection/main.py similarity index 100% rename from weightslab/examples/Usecases/ws-2d-lidar-detection/main.py rename to weightslab/examples/Usecases/wl-2d-lidar-detection/main.py diff --git a/weightslab/examples/Usecases/ws-2d-lidar-detection/utils/criterions.py b/weightslab/examples/Usecases/wl-2d-lidar-detection/utils/criterions.py similarity index 100% rename from weightslab/examples/Usecases/ws-2d-lidar-detection/utils/criterions.py rename to weightslab/examples/Usecases/wl-2d-lidar-detection/utils/criterions.py diff --git a/weightslab/examples/Usecases/ws-2d-lidar-detection/utils/data.py b/weightslab/examples/Usecases/wl-2d-lidar-detection/utils/data.py similarity index 100% rename from weightslab/examples/Usecases/ws-2d-lidar-detection/utils/data.py rename to weightslab/examples/Usecases/wl-2d-lidar-detection/utils/data.py diff --git a/weightslab/examples/Usecases/ws-2d-lidar-detection/utils/model.py b/weightslab/examples/Usecases/wl-2d-lidar-detection/utils/model.py similarity index 100% rename from weightslab/examples/Usecases/ws-2d-lidar-detection/utils/model.py rename to weightslab/examples/Usecases/wl-2d-lidar-detection/utils/model.py diff --git a/weightslab/examples/Usecases/ws-3d-lidar-detection/README.md b/weightslab/examples/Usecases/wl-3d-lidar-detection/README.md similarity index 98% rename from weightslab/examples/Usecases/ws-3d-lidar-detection/README.md rename to weightslab/examples/Usecases/wl-3d-lidar-detection/README.md index 7694bfa8..4d266f14 100644 --- a/weightslab/examples/Usecases/ws-3d-lidar-detection/README.md +++ b/weightslab/examples/Usecases/wl-3d-lidar-detection/README.md @@ -6,8 +6,8 @@ with per-sample and per-instance signals flowing into the WeightsLab dashboards. ``` -ws-3d-lidar-detection/ - main.py # WL wiring + train/eval loop (mirrors PyTorch/ws-detection) +wl-3d-lidar-detection/ + main.py # WL wiring + train/eval loop (mirrors PyTorch/wl-detection) config.yaml # hyperparameters, ranges, loaders utils/ data.py # KITTI-format loader + synthetic scene fallback + collate @@ -18,7 +18,7 @@ ws-3d-lidar-detection/ ## Quick start ```bash -cd weightslab/examples/Usecases/ws-3d-lidar-detection +cd weightslab/examples/Usecases/wl-3d-lidar-detection python main.py ``` @@ -81,7 +81,7 @@ coordinates (`task_type = "detection_pointcloud"` — one task type covering 2D ## WeightsLab signals -Same pattern as `PyTorch/ws-detection`: +Same pattern as `PyTorch/wl-detection`: - `*_loss/sample` (`per_sample=True`) — the YOLO-in-BEV loss, one value per frame: localization (x, y, z) + log-size + sin/cos heading + class CE diff --git a/weightslab/examples/Usecases/ws-3d-lidar-detection/config.yaml b/weightslab/examples/Usecases/wl-3d-lidar-detection/config.yaml similarity index 100% rename from weightslab/examples/Usecases/ws-3d-lidar-detection/config.yaml rename to weightslab/examples/Usecases/wl-3d-lidar-detection/config.yaml diff --git a/weightslab/examples/Usecases/ws-3d-lidar-detection/main.py b/weightslab/examples/Usecases/wl-3d-lidar-detection/main.py similarity index 100% rename from weightslab/examples/Usecases/ws-3d-lidar-detection/main.py rename to weightslab/examples/Usecases/wl-3d-lidar-detection/main.py diff --git a/weightslab/examples/Usecases/ws-3d-lidar-detection/utils/criterions.py b/weightslab/examples/Usecases/wl-3d-lidar-detection/utils/criterions.py similarity index 100% rename from weightslab/examples/Usecases/ws-3d-lidar-detection/utils/criterions.py rename to weightslab/examples/Usecases/wl-3d-lidar-detection/utils/criterions.py diff --git a/weightslab/examples/Usecases/ws-3d-lidar-detection/utils/data.py b/weightslab/examples/Usecases/wl-3d-lidar-detection/utils/data.py similarity index 100% rename from weightslab/examples/Usecases/ws-3d-lidar-detection/utils/data.py rename to weightslab/examples/Usecases/wl-3d-lidar-detection/utils/data.py diff --git a/weightslab/examples/Usecases/ws-3d-lidar-detection/utils/kitti_download.py b/weightslab/examples/Usecases/wl-3d-lidar-detection/utils/kitti_download.py similarity index 100% rename from weightslab/examples/Usecases/ws-3d-lidar-detection/utils/kitti_download.py rename to weightslab/examples/Usecases/wl-3d-lidar-detection/utils/kitti_download.py diff --git a/weightslab/examples/Usecases/ws-3d-lidar-detection/utils/model.py b/weightslab/examples/Usecases/wl-3d-lidar-detection/utils/model.py similarity index 99% rename from weightslab/examples/Usecases/ws-3d-lidar-detection/utils/model.py rename to weightslab/examples/Usecases/wl-3d-lidar-detection/utils/model.py index 0af99b0d..1fc71d27 100644 --- a/weightslab/examples/Usecases/ws-3d-lidar-detection/utils/model.py +++ b/weightslab/examples/Usecases/wl-3d-lidar-detection/utils/model.py @@ -14,7 +14,7 @@ # (objectness, tx, ty, tz, log l, log w, log h, sin yaw, cos yaw, # class_logits...). # -# Encoding (BEV cell-relative, mirrors the 2D ws-detection example): +# Encoding (BEV cell-relative, mirrors the 2D wl-detection example): # * objectness = sigmoid(t_obj) -> P(box centered in cell) # * cx = x_min + (col + sigmoid(tx)) / S * range_x # * cy = y_min + (row + sigmoid(ty)) / S * range_y diff --git a/weightslab/examples/Usecases/wl-classification-signals_shape_classification/config.yaml b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/config.yaml new file mode 100644 index 00000000..97f6d760 --- /dev/null +++ b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/config.yaml @@ -0,0 +1,48 @@ +# ============================================================================= +# Configuration for the per-sample loss-shape classification example (MNIST) +# ============================================================================= +# Every value below can be overridden by the matching WL_STRESS_* / WL_* env +# var (see the env table below) so the example stays scriptable for stress runs. + +# Global +experiment_name: signals-mnist +device: auto # auto | cpu | cuda + +# Run parameters +epochs: 10 # env override: WL_STRESS_EPOCHS +out: /tmp/wl_stress # env override: WL_STRESS_OUT — wiped & recreated on start + # (holds data/, wl_logs/, metrics.jsonl, report.csv) +batch_size: 64 + +# Signals +loss_signal_name: loss_sample +shape_every: 1 # env override: WL_SHAPE_EVERY — throttle for + # sig/loss_shape (reads history, so it's costly). + # 5 = every step (full coverage); higher = cheaper. +min_step: 0 # min step to start computing + # sig/loss_shape (otherwise it will be NaN). + +# Serving +serving_grpc: true +serving_cli: true + +# Ledger / dataframe storage +ledger_flush_max_rows: 8192 +ledger_enable_h5_persistence: false +experiment_dump_to_train_steps_ratio: 10000000 +query_cache_maxsize: 2048 # env override: WL_QUERY_CACHE_MAXSIZE — LRU maxsize + # of the per-sample query cache the ledger uses when + # serving signals (larger = more cached, more memory). + +# Optimizer +optimizer: + lr: 0.01 + +# Data loaders +data: + train_loader: + batch_size: 64 + shuffle: true + test_loader: + batch_size: 256 + shuffle: false diff --git a/weightslab/examples/Usecases/wl-classification-signals_shape_classification/main.py b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/main.py new file mode 100644 index 00000000..8c8de2c5 --- /dev/null +++ b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/main.py @@ -0,0 +1,252 @@ +"""Per-sample signals on MNIST — readable, zero save_signals, with train + eval. + +Parameters live in ``config.yaml`` (loaded at startup). A handful of knobs can +also be overridden by environment variables for scripted stress runs. + +Per-step user code is just the watched loss. Everything else is a @wl.signal +(defined in ``utils/signals.py``): + entropy from ctx.logits when the loss fires + loss_norm reactive, from the logged loss + hardness reactive, from loss + entropy + loss_shape reactive, classifies each sample's loss trajectory (live signal, + not an end-of-run function). Reads history, so it's throttled by + shape_every (1 = every step, full coverage; higher = cheaper). + +Universal loss: the watched crit runs on the test split each epoch too, so test +samples get a loss trajectory and a shape as well. + +Env overrides (else the config.yaml value is used): + WL_STRESS_EPOCHS (int) number of training epochs. + WL_STRESS_OUT (str) output dir; wiped and recreated on start. Holds + data/, wl_logs/, metrics.jsonl and report.csv. + WL_SHAPE_EVERY (int) throttle for the sig/loss_shape signal, which + reads history and is therefore costly. 1 = compute every + step (full coverage); higher = every N steps (cheaper). + WL_QUERY_CACHE_MAXSIZE (int) backend tuning knob (read in + weightslab.backend.logger). Sets the LRU maxsize of the + per-sample query cache the ledger uses when serving + signals; larger values cache more distinct per-sample + queries at the cost of memory. +""" +import os, time, gc, json, shutil +import yaml +import torch +import numpy as np +import pandas as pd +import torch.nn as nn +import torch.optim as optim + +import weightslab as wl + +from collections import Counter + +from weightslab import guard_training_context, guard_testing_context + +from utils.model import SmallCNN +from utils.data import MNISTIdx +from utils.criterions import SHAPES +from utils.signals import register_signals + + +# ============================================================================= +# Configuration (config.yaml + env overrides, see module docstring) +# ============================================================================= +def load_config(): + """Load config.yaml next to this file and apply defaults for missing keys.""" + config_path = os.path.join(os.path.dirname(__file__), "config.yaml") + if os.path.exists(config_path): + with open(config_path, "r") as fh: + cfg = yaml.safe_load(fh) or {} + else: + cfg = {} + + cfg.setdefault("experiment_name", "signals-mnist") + cfg.setdefault("device", "auto") + cfg.setdefault("epochs", 10) + cfg.setdefault("out", "/tmp/wl_stress") + cfg.setdefault("batch_size", 64) + cfg.setdefault("loss_signal_name", "loss_sample") + cfg.setdefault("shape_every", 1) + cfg.setdefault("serving_grpc", False) + cfg.setdefault("serving_cli", False) + cfg.setdefault("ledger_flush_max_rows", 8192) + cfg.setdefault("ledger_enable_h5_persistence", False) + cfg.setdefault("experiment_dump_to_train_steps_ratio", 10_000_000) + cfg.setdefault("query_cache_maxsize", 2048) + cfg.setdefault("optimizer", {}).setdefault("lr", 0.01) + data = cfg.setdefault("data", {}) + data.setdefault("train_loader", {}).setdefault("batch_size", cfg["batch_size"]) + cfg["data"]["train_loader"].setdefault("shuffle", True) + data.setdefault("test_loader", {}).setdefault("batch_size", 256) + cfg["data"]["test_loader"].setdefault("shuffle", False) + + # Env overrides for scripted stress runs (env wins over the yaml value). + cfg["epochs"] = int(os.environ.get("WL_STRESS_EPOCHS", cfg["epochs"])) + cfg["out"] = os.environ.get("WL_STRESS_OUT", cfg["out"]) + cfg["shape_every"] = int(os.environ.get("WL_SHAPE_EVERY", cfg["shape_every"])) + cfg["min_step"] = int(os.environ.get("WL_MIN_STEP", cfg["min_step"])) + cfg["query_cache_maxsize"] = int( + os.environ.get("WL_QUERY_CACHE_MAXSIZE", cfg["query_cache_maxsize"])) + return cfg + + +def resolve_device(name): + if name == "auto": + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + return torch.device(name) + + +# ============================================================================= +# Helpers +# ============================================================================= +def rss_gb(): + try: + for ln in open("/proc/self/status"): + if ln.startswith("VmRSS:"): + return int(ln.split()[1]) / (1024 * 1024) + except Exception: + return -1.0 + + +def log(msg): + print(msg, flush=True) + + +def vanilla_baseline(dev, batch, lr, sync): + """Plain-PyTorch step time (ms) for the WeightsLab overhead comparison.""" + vb = SmallCNN().to(dev); vo = optim.Adam(vb.parameters(), lr=lr); vc = nn.CrossEntropyLoss() + xb = torch.randn(batch, 1, 28, 28, device=dev); yb = torch.randint(0, 10, (batch,), device=dev) + for _ in range(10): + vo.zero_grad(); vc(vb(xb), yb).backward(); vo.step() + sync(); t0 = time.perf_counter() + for _ in range(50): + vo.zero_grad(); vc(vb(xb), yb).backward(); vo.step() + sync(); vanilla_ms = 1000 * (time.perf_counter() - t0) / 50 + del vb, vo; gc.collect() + return vanilla_ms + + +# ============================================================================= +# Train / Test loops (classification, using watcher-wrapped loaders) +# ============================================================================= +def train(loader, model, opt, crit, dev, sync): + """One training epoch. Returns the list of per-step wall times (ms). + + The only per-step call is the watched loss: it logs loss_sample and fires + the @wl.signal chain. No save_signals. + """ + ep_ms = [] + for img, ids, lab in loader: + img, lab = img.to(dev), lab.to(dev) + sync(); ts = time.perf_counter() + with guard_training_context: + opt.zero_grad() + logits = model(img) + crit(logits, lab, batch_ids=ids, preds=logits.argmax(1, keepdim=True)).mean().backward() + opt.step() + sync(); ep_ms.append(1000 * (time.perf_counter() - ts)) + return ep_ms + + +def test(test_loader, model, crit, dev): + """Universal loss: run the watched crit over the whole test split (one pass). + + Test samples get a loss trajectory and a shape too. + """ + with torch.no_grad(): + for tb in test_loader: + ti, tid, tl = tb[0].to(dev), tb[1], tb[2].to(dev) + with guard_testing_context: + tlg = model(ti) + crit(tlg, tl, batch_ids=tid, preds=tlg.argmax(1, keepdim=True)) + + +# ============================================================================= +# Main +# ============================================================================= +def main(): + # --- 1) Config: load, resolve device, prepare output dir --- + cfg = load_config() + LOSS = cfg["loss_signal_name"] + OUT = cfg["out"] + EPOCHS = cfg["epochs"] + SHAPE_EVERY = cfg["shape_every"] + SHAPE_MIN_STEP = cfg["min_step"] + BATCH = int(cfg["data"]["train_loader"]["batch_size"]) + LR = float(cfg["optimizer"]["lr"]) + # Backend reads this from the environment when it builds the query cache. + os.environ["WL_QUERY_CACHE_MAXSIZE"] = str(cfg["query_cache_maxsize"]) + + shutil.rmtree(OUT, ignore_errors=True) + os.makedirs(OUT + "/wl_logs", exist_ok=True) + dev = resolve_device(cfg["device"]) + torch.manual_seed(0) + metrics = open(OUT + "/metrics.jsonl", "w") + sync = (lambda: torch.cuda.synchronize()) if dev.type == "cuda" else (lambda: None) + + # --- 2) Vanilla baseline (plain PyTorch) for the overhead comparison --- + vanilla_ms = vanilla_baseline(dev, BATCH, LR, sync) + log(f"[run] vanilla baseline = {vanilla_ms:.2f} ms/step (dev {dev})") + + # --- 3) WeightsLab setup: both splits tracked, watched loss --- + hp = {"experiment_name": cfg["experiment_name"], "device": str(dev), + "root_log_dir": OUT + "/wl_logs", + "serving_grpc": cfg["serving_grpc"], "serving_cli": cfg["serving_cli"], + "ledger_flush_max_rows": cfg["ledger_flush_max_rows"], + "ledger_enable_h5_persistence": cfg["ledger_enable_h5_persistence"], + "experiment_dump_to_train_steps_ratio": cfg["experiment_dump_to_train_steps_ratio"], + "data": {"train_loader": {"batch_size": BATCH, + "shuffle": cfg["data"]["train_loader"]["shuffle"]}}} + wl.watch_or_edit(hp, flag="hyperparameters", defaults=hp) + train_ds = MNISTIdx(OUT + "/data", train=True, base=0) + test_ds = MNISTIdx(OUT + "/data", train=False, base=1_000_000) + model = wl.watch_or_edit(SmallCNN().to(dev), flag="model", device=dev) + opt = wl.watch_or_edit(optim.Adam(model.parameters(), lr=LR), flag="optimizer") + loader = wl.watch_or_edit(train_ds, flag="data", loader_name="train_loader", + batch_size=BATCH, + shuffle=cfg["data"]["train_loader"]["shuffle"], + is_training=True, preload_labels=True) + test_loader = wl.watch_or_edit(test_ds, flag="data", loader_name="test_loader", + batch_size=int(cfg["data"]["test_loader"]["batch_size"]), + shuffle=cfg["data"]["test_loader"]["shuffle"], + is_training=False, preload_labels=True) + crit = wl.watch_or_edit(nn.CrossEntropyLoss(reduction="none"), + flag="loss", signal_name=LOSS, per_sample=True, log=True) + + # --- 4) Per-sample signals (the @wl.signal chain, see utils/signals.py) --- + register_signals(LOSS, SHAPE_EVERY, SHAPE_MIN_STEP) + + # --- 5) Serve + launch training --- + wl.serve(serving_grpc=cfg["serving_grpc"], serving_cli=cfg["serving_cli"]) + wl.start_training(timeout=5) # Let WeightsLab Initialize + log(f"[run] tracked train={len(train_ds)} test={len(test_ds)} | shape_every={SHAPE_EVERY}") + + # --- 6) Train / eval loop --- + step_times, gstep, t_run = [], 0, time.perf_counter() + for ep in range(1, EPOCHS + 1): + ep_ms = train(loader, model, opt, crit, dev, sync) + gstep += len(ep_ms) + test(test_loader, model, crit, dev) + wl_ms = float(np.mean(ep_ms)); step_times += ep_ms + rec = {"epoch": ep, "gstep": gstep, "wl_ms": round(wl_ms, 2), "vanilla_ms": round(vanilla_ms, 2), + "rss_gb": round(rss_gb(), 2), "elapsed_s": round(time.perf_counter() - t_run, 1)} + metrics.write(json.dumps(rec) + "\n"); metrics.flush() + log(f"[run] ep {ep:3d}/{EPOCHS} | WL {wl_ms:6.2f} ms vs vanilla {vanilla_ms:.2f} " + f"= +{100*(wl_ms/vanilla_ms-1):.0f}% | RSS {rec['rss_gb']:.2f} GB | {rec['elapsed_s']:.0f}s") + + # --- 7) Report + summary --- + path = wl.write_dataframe(OUT + "/report.csv", format="csv", columns="signals") + df = pd.read_csv(path) + sc = [c for c in df.columns if c.endswith("sig/loss_shape")][0] + dist = Counter(SHAPES[int(v)] for v in df[sc].dropna() if v >= 0) + med = float(np.median(step_times)) + metrics.close() + log("\n[run] ===== SUMMARY =====") + log(f"[run] vanilla {vanilla_ms:.2f} ms | WL median {med:.2f} ms/step (+{100*(med/vanilla_ms-1):.0f}%)") + log(f"[run] report {len(df)} rows | loss_shape covered {df[sc].notna().sum()}/{len(df)}") + log(f"[run] shape distribution: {dict(dist)}") + log(f"[run] report -> {path}") + + +if __name__ == "__main__": + main() diff --git a/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/criterions.py b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/criterions.py new file mode 100644 index 00000000..0837832a --- /dev/null +++ b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/criterions.py @@ -0,0 +1,36 @@ +# ============================================================================= +# Loss-shape classification +# ============================================================================= +# The watched criterion is a stock ``nn.CrossEntropyLoss(reduction="none")`` +# wrapped per-sample in main.py. This module holds the loss-*trajectory* +# classifier used by the ``sig/loss_shape`` signal: given each sample's loss +# history it labels the curve as one of ``SHAPES``. +import numpy as np + +SHAPES = ["monotonic", "plateaued", "Flat_high", "high_variance", "U_Shape", "Spiked"] + + +def classify_shape(values): + """Loss trajectory -> shape index (or -1 with < 5 points).""" + y = np.asarray(values, dtype=float) + if y.size < 5: + return -1 + n = y.size + rng = max(float(y.max() - y.min()), 1e-8) + drop = (float(y[0]) - float(y[-1])) / (abs(float(y[0])) + 1e-8) + cv = float(y.std()) / (abs(float(y.mean())) + 1e-8) + argmin = int(np.argmin(y)) + rebound = (float(y[-1]) - float(y.min())) / rng + tail = y[int(0.6 * n):] + tail_flat = float(tail.std()) / (abs(float(tail.mean())) + 1e-8) < 0.1 + if 0.2 * n < argmin < 0.8 * n and rebound > 0.3: + return SHAPES.index("U_Shape") + if drop > 0.4: + return SHAPES.index("monotonic") + if drop > 0.15 and tail_flat: + return SHAPES.index("plateaued") + if float(np.diff(y).max()) / rng > 0.5: + return SHAPES.index("Spiked") + if cv > 0.5: + return SHAPES.index("high_variance") + return SHAPES.index("Flat_high") diff --git a/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/data.py b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/data.py new file mode 100644 index 00000000..7f23157d --- /dev/null +++ b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/data.py @@ -0,0 +1,23 @@ +# ============================================================================= +# MNIST dataset yielding per-sample uids for the shared WeightsLab ledger +# ============================================================================= +from torch.utils.data import Dataset +from torchvision import datasets, transforms + + +class MNISTIdx(Dataset): + """Yields (image, uid, label). uid namespaced by split so train/test don't + collide in the shared ledger; fast_get_label skips decode at ledger init.""" + def __init__(self, root, train, base): + self.m = datasets.MNIST(root, train=train, download=True, transform=None) + self.t = transforms.ToTensor(); self.base = base + + def __len__(self): + return len(self.m) + + def __getitem__(self, i): + img, lab = self.m[i] + return self.t(img), self.base + i, lab + + def fast_get_label(self, i): + return int(self.m.targets[i]) diff --git a/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/model.py b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/model.py new file mode 100644 index 00000000..311585e7 --- /dev/null +++ b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/model.py @@ -0,0 +1,19 @@ +# ============================================================================= +# Small CNN classifier for MNIST +# ============================================================================= +# Two conv/pool blocks into a small MLP head producing 10 class logits. The +# ``input_shape`` attribute lets WeightsLab introspect the model. +import torch.nn as nn + + +class SmallCNN(nn.Module): + def __init__(self): + super().__init__() + self.input_shape = (1, 1, 28, 28) + self.net = nn.Sequential( + nn.Conv2d(1, 8, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Conv2d(8, 16, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Flatten(), nn.Linear(16 * 7 * 7, 64), nn.ReLU(), nn.Linear(64, 10)) + + def forward(self, x): + return self.net(x) diff --git a/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/signals.py b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/signals.py new file mode 100644 index 00000000..696ceb10 --- /dev/null +++ b/weightslab/examples/Usecases/wl-classification-signals_shape_classification/utils/signals.py @@ -0,0 +1,48 @@ +# ============================================================================= +# Per-sample @wl.signal chain +# ============================================================================= +# Per-step user code is just the watched loss. Everything here is reactive: +# sig/entropy from the logits when the watched loss fires +# sig/loss_norm batch-normalized loss (loss / mean(loss)) +# sig/hardness loss * entropy +# sig/loss_shape classifies each sample's loss trajectory (live signal, not an +# end-of-run function). Reads history, so it's throttled by +# shape_every (1 = every step; higher = cheaper, sparser). +import numpy as np +import torch + +import weightslab as wl + +from utils.criterions import classify_shape + + +def register_signals(loss_name, shape_every, min_step: int = 0): + """Define and register the per-sample signal chain on the watched loss. + + Defining a ``@wl.signal`` registers it globally, so this must be called + before ``wl.serve`` / ``wl.start_training``. + + Args: + loss_name: name of the watched per-sample loss signal to subscribe to. + shape_every: compute sig/loss_shape every N steps (throttle). + min_step: minimum step to start computing sig/loss_shape. + """ + + @wl.signal(name="sig/entropy", subscribe_to=loss_name, batched=True) + def entropy(b): + p = torch.softmax(b.logits, 1) + return (-(p * (p + 1e-12).log()).sum(1)).detach().cpu().numpy() + + @wl.signal(name="sig/loss_norm", inputs=[loss_name], batched=True) + def loss_norm(b): + return b.inputs[loss_name] / (float(np.mean(b.inputs[loss_name])) + 1e-8) + + @wl.signal(name="sig/hardness", inputs=[loss_name, "sig/entropy"], batched=True) + def hardness(b): + return b.inputs[loss_name] * b.inputs["sig/entropy"] + + @wl.signal(name="sig/loss_shape", inputs=[loss_name], batched=True, + compute_every_n_steps=shape_every, min_step=min_step) + def loss_shape(b): + hist = b.history(loss_name) # {uid: [loss values in step order]} + return np.array([classify_shape(hist[s]) for s in b.sample_ids], dtype=float) diff --git a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/config.yaml b/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/config.yaml deleted file mode 100644 index 26c6ae44..00000000 --- a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Global configuration -device: auto -experiment_name: detection_pennfudan_usecase -training_steps_to_do: null # null = infinite training until manually stopped (behavior set by the user in main script) -# root_log_dir: # Empty to write in tmp directory, or specify a path to store logs and checkpoints - -checkpoint_manager: - load_config: false # Disable loading previous checkpoint configuration to allow for changes in the current config.yaml file between experiments. Otherwise changes will be replaced. - -# Initially compute natural sorting values, e.g., average intensity. More details in the README.md file. -compute_natural_sort: true - -# Experiment parameters -eval_full_to_train_steps_ratio: 5 -experiment_dump_to_train_steps_ratio: 5 -tqdm_display: true -is_training: false # Start training immediately or not - -# Serving -serving_grpc: true -serving_cli: true - -# Configure global dataframe storage -ledger_enable_h5_persistence: true -ledger_enable_flushing_threads: true -ledger_flush_max_rows: 100 -ledger_flush_interval: 60.0 - -# Model / task -num_classes: 1 # Penn-Fudan: single class (person) -image_size: 256 -grid_size: 8 # detector predicts on an 8x8 cell grid -conf_thresh: 0.3 # objectness*class confidence threshold for displayed predictions -pretrained_backbone: true # ImageNet-pretrained MobileNetV3-Small feature extractor -freeze_backbone: true # train only the detection head (faster, less data hungry) - -optimizer: - lr: 0.001 - -# Data (Penn-Fudan pedestrians; ~170 images downloaded on first run under data_root) -data_root: .\data -data: - train_loader: - batch_size: 8 - shuffle: true - max_samples: null # null = use the full training split - test_loader: - batch_size: 8 - shuffle: false - max_samples: null diff --git a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/main.py b/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/main.py deleted file mode 100644 index ff4b2030..00000000 --- a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/main.py +++ /dev/null @@ -1,340 +0,0 @@ -import os -import time -import numpy as np -import tqdm -import yaml -import torch -import logging -import tempfile -import itertools - -import weightslab as wl - -from torch import optim - - -from weightslab.components.global_monitoring import ( - guard_training_context, - guard_testing_context, -) - -from utils.data import PennFudanDetectionDataset, det_collate -from utils.model import SmallDetector -from utils.criterions import ( - PerSampleDetectionLoss, - PerSampleIoU, - PerInstanceIoU, - decode_predictions, -) - -# Setup loggers -logging.basicConfig(level=logging.ERROR) -logging.getLogger("PIL").setLevel(logging.INFO) - - -# ============================================================================= -# Train / Test loops (detection, using watcher-wrapped loaders) -# ============================================================================= - -def train(loader, model, optimizer, sig, device, grid_size, conf_thresh): - """Single training step using the tracked dataloader + watched loss. - - loader yields (inputs, ids, targets, metadata) because of the - DataSampleTrackingWrapper. `targets` is per sample a [N, 6] tensor of boxes - ([x1, y1, x2, y2, class_id, confidence]); see utils/data.det_collate. - """ - with guard_training_context: - (inputs, ids, targets, _) = next(loader) - inputs = inputs.to(device) - targets = [t.to(device) for t in targets] - - optimizer.zero_grad() - outputs = model(inputs) # [B, S, S, 5 + num_classes] - - # Decoded boxes for the UI overlay (detached — display only). - preds = decode_predictions(outputs.detach(), grid_size, conf_thresh=conf_thresh) - - # Per-sample detection loss (tracked, saved, and the value we backprop). - # `preds=` rides along so WL stores the predicted boxes for this batch. - loss_per_sample = sig["loss"](outputs, targets, batch_ids=ids, preds=preds) - - # Metrics: per-sample mean IoU + per-instance IoU (one value per box). - sig["iou_sample"](outputs, targets, batch_ids=ids) - sig["iou_instance"](outputs, targets, batch_ids=ids) - - loss = loss_per_sample.mean() - loss.backward() - optimizer.step() - - return float(loss.detach().cpu().item()) - - -def test(loader, model, sig, device, grid_size, conf_thresh, test_loader_len): - """Full evaluation pass over the val loader.""" - losses = 0.0 - ious = 0.0 - with guard_testing_context, torch.no_grad(): - for inputs, ids, targets, _ in loader: - inputs = inputs.to(device) - targets = [t.to(device) for t in targets] - - outputs = model(inputs) - preds = decode_predictions(outputs, grid_size, conf_thresh=conf_thresh) - - loss_per_sample = sig["loss"](outputs, targets, batch_ids=ids, preds=preds) - iou_per_sample = sig["iou_sample"](outputs, targets, batch_ids=ids) - sig["iou_instance"](outputs, targets, batch_ids=ids) - - losses += torch.mean(loss_per_sample) - ious += torch.mean(iou_per_sample) - - loss = float((losses / test_loader_len).detach().cpu().item()) - iou = float((ious / test_loader_len).detach().cpu().item()) - return loss, iou * 100.0 # Return mean IoU as percentage - - -# ============================================================================= -# Main -# ============================================================================= -if __name__ == "__main__": - # --- 1) Load hyperparameters from YAML (if present) --- - config_path = os.path.join(os.path.dirname(__file__), "config.yaml") - if os.path.exists(config_path): - with open(config_path, "r") as fh: - parameters = yaml.safe_load(fh) or {} - else: - parameters = {} - - # Defaults - parameters.setdefault("experiment_name", "pennfudan_detection") - parameters.setdefault("device", "auto") - parameters.setdefault("training_steps_to_do", 500) - parameters.setdefault("eval_full_to_train_steps_ratio", 50) - parameters.setdefault("number_of_workers", 4) - parameters.setdefault("num_classes", 1) # Penn-Fudan: single class (person) - parameters.setdefault("image_size", 256) - parameters.setdefault("grid_size", 8) - parameters.setdefault("conf_thresh", 0.3) - parameters.setdefault("pretrained_backbone", True) - parameters.setdefault("freeze_backbone", True) - parameters.setdefault("compute_natural_sort", True) - - # --- 2) Register hyperparameters --- - exp_name = parameters["experiment_name"] - wl.watch_or_edit( - parameters, - flag="hyperparameters", - name=exp_name, - defaults=parameters, - poll_interval=1.0, - ) - - num_classes = int(parameters["num_classes"]) - image_size = int(parameters["image_size"]) - grid_size = int(parameters["grid_size"]) - conf_thresh = float(parameters["conf_thresh"]) - - # --- 3) Device selection --- - if parameters.get("device", "auto") == "auto": - parameters["device"] = torch.device( - "cuda" if torch.cuda.is_available() else "cpu" - ) - device = parameters["device"] - - # --- 4) Logging directory --- - if not parameters.get("root_log_dir"): - tmp_dir = tempfile.mkdtemp() - parameters["root_log_dir"] = tmp_dir - print(f"No root_log_dir specified, using temporary directory: {parameters['root_log_dir']}") - os.makedirs(parameters["root_log_dir"], exist_ok=True) - log_dir = parameters["root_log_dir"] - max_steps = parameters["training_steps_to_do"] - eval_full_to_train_steps_ratio = parameters["eval_full_to_train_steps_ratio"] - verbose = parameters.get("verbose", True) - tqdm_display = parameters.get("tqdm_display", True) - - - # --- 5) Data (Penn-Fudan pedestrians, downloaded on first run) --- - default_data_root = os.path.abspath( - os.path.join(os.path.dirname(__file__), "data") - ) - data_root = parameters.get("data_root", default_data_root) - - train_cfg = parameters.get("data", {}).get("train_loader", {}) - test_cfg = parameters.get("data", {}).get("test_loader", {}) - - _train_dataset = PennFudanDetectionDataset( - root=data_root, - split="train", - num_classes=num_classes, - image_size=image_size, - max_samples=train_cfg.get("max_samples", None), - ) - _val_dataset = PennFudanDetectionDataset( - root=data_root, - split="val", - num_classes=num_classes, - image_size=image_size, - max_samples=test_cfg.get("max_samples", None), - ) - - train_loader = wl.watch_or_edit( - _train_dataset, - flag="data", - loader_name="train_loader", - batch_size=train_cfg.get("batch_size", 8), - shuffle=train_cfg.get("shuffle", True), - compute_hash=False, - is_training=True, - array_autoload_arrays=False, - array_return_proxies=True, - array_use_cache=True, - preload_labels=False, - collate_fn=det_collate, - ) - test_loader = wl.watch_or_edit( - _val_dataset, - flag="data", - loader_name="test_loader", - batch_size=test_cfg.get("batch_size", 8), - shuffle=test_cfg.get("shuffle", False), - compute_hash=False, - is_training=False, - array_autoload_arrays=False, - array_return_proxies=True, - array_use_cache=True, - preload_labels=True, - collate_fn=det_collate, - ) - - # --- 6) Model, optimizer, losses, metric --- - _model = SmallDetector( - in_channels=3, num_classes=num_classes, - image_size=image_size, grid_size=grid_size, - pretrained=bool(parameters["pretrained_backbone"]), - freeze_backbone=bool(parameters["freeze_backbone"]), - ).to(device) - model = wl.watch_or_edit( - _model, - flag="model", - device=device - ) - lr = parameters.get("optimizer", {}).get("lr", 1e-3) - # Only optimize trainable params (the head; backbone may be frozen). - trainable = [p for p in model.parameters() if p.requires_grad] - _optimizer = optim.Adam(trainable, lr=lr) - optimizer = wl.watch_or_edit( - _optimizer, - flag="optimizer", - ) - - # --- Detection loss (per sample) + IoU (per sample & per instance) signals --- - # per_sample=True auto-saves one value per sample; per_instance=True auto-saves - # one IoU per (sample_id, annotation_id), i.e. one per ground-truth box. - def _make_det_signals(split: str, weights=None) -> dict: - return { - "loss": wl.watch_or_edit( - PerSampleDetectionLoss(num_classes, grid_size, weights=weights), - flag="loss", - name=f"{split}_loss/sample", per_sample=True, log=True, - ), - "iou_sample": wl.watch_or_edit( - PerSampleIoU(num_classes, grid_size), flag="metric", - name=f"{split}_iou/sample", per_sample=True, log=True, - ), - "iou_instance": wl.watch_or_edit( - PerInstanceIoU(num_classes, grid_size), flag="metric", - name=f"{split}_iou/instance", per_instance=True, log=True, - ), - } - - # Class weights from ground-truth box counts (optional; balances rare shapes). - def compute_class_weights(dataset, num_classes, max_samples=200): - print("\n" + "=" * 60, flush=True) - print(f"Computing class weights for {num_classes} classes (max {max_samples} samples)...", flush=True) - class_counts = np.zeros(num_classes, dtype=np.float64) - num_samples = min(len(dataset), max_samples) - - for idx in tqdm.tqdm(range(num_samples), desc=" Analyzing Distribution"): - _, _, target, _ = dataset.get_items(idx, include_labels=True) - if target is None or len(target) == 0: - continue - for c in target[:, 4].astype(np.int64): - if 0 <= c < num_classes: - class_counts[c] += 1 - - class_counts = np.maximum(class_counts, 1) # Avoid div by zero - total = class_counts.sum() - class_weights = total / (num_classes * class_counts) - class_weights = class_weights / class_weights.mean() # Normalize - - print("\nClass distribution and weights:", flush=True) - for c in range(num_classes): - pct = (class_counts[c] / total) * 100 - print(f"Class {c} ({dataset.class_names[c]}): {pct:6.2f}% -> weight: {class_weights[c]:.3f}", flush=True) - print("=" * 60 + "\n", flush=True) - return torch.FloatTensor(class_weights).to(device) - - weights = compute_class_weights(_train_dataset, num_classes) - - train_sig = _make_det_signals("train", weights=weights) - test_sig = _make_det_signals("test", weights=weights) - - # --- 7) Start WeightsLab services --- - wl.serve( - serving_grpc=parameters.get("serving_grpc", True), - serving_cli=parameters.get("serving_cli", True), - ) - - print("=" * 60) - print(" STARTING PENN-FUDAN PEDESTRIAN DETECTION TRAINING") - print(f" Total steps: {max_steps}") - print(f" Evaluation every {eval_full_to_train_steps_ratio} steps") - print(f" Logs will be saved to: {log_dir}") - print(f" Data root: {data_root}") - print("=" * 60 + "\n") - - # ================ - # Training Loop - wl.start_training(timeout=3) # Blocks and keeps the main thread alive while background services run. Optionally set a timeout (seconds) to auto-stop. - - # ================ - train_range = tqdm.tqdm(itertools.count(), desc="Training") if tqdm_display else itertools.count() - test_loss, test_metric = None, None - start_time = time.time() - for train_step in train_range: - age = model.get_age() if hasattr(model, "get_age") else train_step - - # Train - train_loss = train(train_loader, model, optimizer, train_sig, device, grid_size, conf_thresh) - - # Test - if age == 0 or age % eval_full_to_train_steps_ratio == 0: - test_loader_len = len(test_loader) # Store length before wrapping with tqdm - test_loader_it = tqdm.tqdm(test_loader, desc="Evaluating") if tqdm_display else test_loader - test_loss, test_metric = test(test_loader_it, model, test_sig, device, grid_size, conf_thresh, test_loader_len) - - # Verbose - if verbose and not tqdm_display: - print( - "Training.. " + - f"Step {train_step} (Age {age}): " + - f"| Train Loss: {train_loss:.4f} " + - (f"| Test Loss: {test_loss:.4f} " if test_loss is not None else '') + - (f"| Test IoU: {test_metric:.2f}% " if test_metric is not None else '') - ) - elif tqdm_display: - train_range.set_description("Step") - train_range.set_postfix( - train_loss=f"{train_loss:.4f}", - test_loss=f"{test_loss:.4f}" if test_loss is not None else "N/A", - iou=f"{test_metric:.2f}%" if test_metric is not None else "N/A" - ) - - print("\n" + "=" * 60) - print(f" Training completed in {time.time() - start_time:.2f} seconds") - print(f" Logs saved to: {log_dir}") - print("=" * 60) - - # Keep the main thread alive to allow background serving threads to run - wl.keep_serving() diff --git a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/utils/criterions.py b/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/utils/criterions.py deleted file mode 100644 index 7cd6c9c9..00000000 --- a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/utils/criterions.py +++ /dev/null @@ -1,361 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F - -from .model import decode_grid - -import weightslab as wl -import numpy as np - - -# ============================================================================= -# Per-instance / per-sample detection criterions (YOLO-v1 style loss + IoU) -# ============================================================================= -# The detection dataset yields, per sample, a [N, 6] target tensor -# ``[x1, y1, x2, y2, class_id, confidence]`` (normalized to [0, 1]). Each GT box -# is assigned to the grid cell containing its center; that cell is "responsible" -# for predicting the box. -# -# * PerSampleDetectionLoss -> one differentiable loss scalar per sample ([B]), -# wrapped with ``per_sample=True`` (the value WL backprops + dashboards). -# * PerSampleIoU -> mean IoU over a sample's boxes ([B]), a metric. -# * PerInstanceIoU -> flat tensor of one IoU per GT box (sample-major -# order), wrapped with ``per_instance=True`` so WL auto-saves it at -# (sample_id, annotation_id). The ordering matches the per-sample target -# iteration, so the wrapper's auto ``batch_idx`` maps each value correctly. - -_EPS = 1e-6 -_LAMBDA_COORD = 5.0 -_LAMBDA_NOOBJ = 0.5 - - -def box_iou_xyxy(a, b): - """IoU between two aligned sets of xyxy boxes. a, b: [..., 4] -> [...].""" - x1 = torch.maximum(a[..., 0], b[..., 0]) - y1 = torch.maximum(a[..., 1], b[..., 1]) - x2 = torch.minimum(a[..., 2], b[..., 2]) - y2 = torch.minimum(a[..., 3], b[..., 3]) - - inter = (x2 - x1).clamp(min=0) * (y2 - y1).clamp(min=0) - area_a = (a[..., 2] - a[..., 0]).clamp(min=0) * (a[..., 3] - a[..., 1]).clamp(min=0) - area_b = (b[..., 2] - b[..., 0]).clamp(min=0) * (b[..., 3] - b[..., 1]).clamp(min=0) - union = area_a + area_b - inter + _EPS - return inter / union - - -def _responsible_cells(boxes, S): - """Map GT boxes -> their responsible (row, col) cell and center offsets. - - Args: - boxes: [N, 4] xyxy in [0, 1]. - S: grid size. - - Returns: - rows, cols: [N] long, the responsible cell indices. - off_x, off_y: [N] center offset within the cell, in [0, 1). - w, h: [N] box size as a fraction of the image. - """ - cx = (boxes[:, 0] + boxes[:, 2]) / 2 - cy = (boxes[:, 1] + boxes[:, 3]) / 2 - w = (boxes[:, 2] - boxes[:, 0]).clamp(_EPS, 1.0) - h = (boxes[:, 3] - boxes[:, 1]).clamp(_EPS, 1.0) - - cols = (cx * S).long().clamp(0, S - 1) - rows = (cy * S).long().clamp(0, S - 1) - off_x = (cx * S - cols).clamp(0, 1) - off_y = (cy * S - rows).clamp(0, 1) - return rows, cols, off_x, off_y, w, h - - -def _per_sample_loss(outputs, targets, num_classes, weights=None): - """YOLO-v1 style loss, returned as one scalar per sample ([B], with grad).""" - B, S = outputs.shape[0], outputs.shape[1] - device = outputs.device - - obj_logit = outputs[..., 0] # [B, S, S] - tx = torch.sigmoid(outputs[..., 1]) - ty = torch.sigmoid(outputs[..., 2]) - w_pred = torch.sigmoid(outputs[..., 3]) - h_pred = torch.sigmoid(outputs[..., 4]) - cls_logits = outputs[..., 5:] # [B, S, S, C] - - if weights is not None: - weights = torch.as_tensor(weights, device=device, dtype=outputs.dtype) - - losses = [] - for s in range(B): - tgt = targets[s] - tgt = torch.as_tensor(tgt, device=device, dtype=outputs.dtype) - if tgt.ndim == 1: - tgt = tgt.view(-1, 6) if tgt.numel() else tgt.view(0, 6) - - obj_target = torch.zeros((S, S), device=device, dtype=outputs.dtype) - - coord_loss = torch.zeros((), device=device) - class_loss = torch.zeros((), device=device) - - if tgt.numel() > 0: - boxes = tgt[:, :4] - cls_ids = tgt[:, 4].long().clamp(0, num_classes - 1) - rows, cols, off_x, off_y, gw, gh = _responsible_cells(boxes, S) - - obj_target[rows, cols] = 1.0 - - # Localization: center offset (linear) + size in sqrt space (YOLO trick - # so small-box errors weigh as much as large-box ones). - coord = ( - (tx[s, rows, cols] - off_x) ** 2 - + (ty[s, rows, cols] - off_y) ** 2 - + (torch.sqrt(w_pred[s, rows, cols] + _EPS) - torch.sqrt(gw + _EPS)) ** 2 - + (torch.sqrt(h_pred[s, rows, cols] + _EPS) - torch.sqrt(gh + _EPS)) ** 2 - ) - coord_loss = _LAMBDA_COORD * coord.sum() - - ce = F.cross_entropy( - cls_logits[s, rows, cols], cls_ids, reduction="none" - ) - if weights is not None: - ce = ce * weights[cls_ids] - class_loss = ce.sum() - - # Objectness BCE over the whole grid; down-weight the (many) empty cells. - obj_weight = torch.where( - obj_target > 0, - torch.ones_like(obj_target), - torch.full_like(obj_target, _LAMBDA_NOOBJ), - ) - obj_loss = ( - F.binary_cross_entropy_with_logits( - obj_logit[s], obj_target, reduction="none" - ) * obj_weight - ).sum() - - losses.append(coord_loss + class_loss + obj_loss) - - return torch.stack(losses) - - -def _per_box_iou(outputs, targets, grid_size): - """IoU of each GT box against its responsible cell's decoded prediction. - - Returns a list[B] of 1-D tensors (one IoU per box for that sample, in - annotation order). Detached — this is a metric, not a loss. - """ - boxes_grid, _, _ = decode_grid(outputs, grid_size) # [B, S, S, 4] - B = outputs.shape[0] - S = grid_size - device = outputs.device - - per_sample = [] - for s in range(B): - tgt = torch.as_tensor(targets[s], device=device, dtype=outputs.dtype) - if tgt.numel() == 0: - per_sample.append(torch.zeros(0, device=device)) - continue - if tgt.ndim == 1: - tgt = tgt.view(-1, 6) - - gt_boxes = tgt[:, :4] - rows, cols, _, _, _, _ = _responsible_cells(gt_boxes, S) - pred_boxes = boxes_grid[s, rows, cols] # [N, 4] - ious = box_iou_xyxy(pred_boxes, gt_boxes) # [N] - per_sample.append(ious.detach()) - - return per_sample - - -class PerSampleDetectionLoss(nn.Module): - """Total detection loss aggregated to one value per sample ([B]).""" - - def __init__(self, num_classes, grid_size, weights=None): - super().__init__() - self.num_classes = num_classes - self.grid_size = grid_size - self.register_buffer( - "weights", torch.tensor(weights) if weights is not None else None - ) - - def forward(self, outputs, targets): - return _per_sample_loss(outputs, targets, self.num_classes, weights=self.weights) - - -class PerSampleIoU(nn.Module): - """Mean IoU over a sample's boxes -> one value per sample ([B]).""" - - def __init__(self, num_classes, grid_size): - super().__init__() - self.num_classes = num_classes - self.grid_size = grid_size - - def forward(self, outputs, targets): - per_sample = _per_box_iou(outputs, targets, self.grid_size) - out = [ - v.mean() if v.numel() > 0 else torch.zeros((), device=outputs.device) - for v in per_sample - ] - return torch.stack(out).detach() - - -class PerInstanceIoU(nn.Module): - """IoU per GT box -> flat tensor [total_boxes] (sample-major order).""" - - def __init__(self, num_classes, grid_size): - super().__init__() - self.num_classes = num_classes - self.grid_size = grid_size - - def forward(self, outputs, targets): - per_sample = _per_box_iou(outputs, targets, self.grid_size) - flat = [v for s in per_sample for v in s] - if not flat: - return torch.zeros(0, device=outputs.device) - return torch.stack(flat).detach() - - -# ============================================================================= -# Inference-time decoding (for UI prediction overlays) -# ============================================================================= -def decode_predictions(outputs, grid_size, conf_thresh=0.3, max_det=10): - """Turn raw grid logits into a per-sample list of detected boxes. - - Returns list[B] of [M, 6] numpy-friendly tensors - ``[x1, y1, x2, y2, class_id, confidence]`` (kept on CPU, detached) — the - exact 6-column schema WL renders for detection predictions. - """ - boxes_grid, obj, cls_probs = decode_grid(outputs, grid_size) - B, S = outputs.shape[0], grid_size - - cls_conf, cls_id = cls_probs.max(dim=-1) # [B, S, S] - score = obj * cls_conf # combined confidence - - flat_boxes = boxes_grid.view(B, S * S, 4) - flat_score = score.view(B, S * S) - flat_cls = cls_id.view(B, S * S) - - results = [] - for s in range(B): - keep = flat_score[s] >= conf_thresh - if keep.sum() == 0: - results.append(torch.zeros((0, 6))) - continue - sc = flat_score[s][keep] - bx = flat_boxes[s][keep] - cl = flat_cls[s][keep].to(bx.dtype) - - # Keep the most confident detections (cheap top-k in place of full NMS). - order = torch.argsort(sc, descending=True)[:max_det] - det = torch.cat([bx[order], cl[order, None], sc[order, None]], dim=1) - results.append(det.detach().cpu()) - - return results - - - - -# ========================================================================= -# Custom subscribed signal: per-sample loss-shape classification -# ========================================================================= -# This is a *dynamic* WeightsLab signal. It subscribes to the per-sample -# classification loss "train/clsf_sample" and, every 25 optimisation steps, -# inspects each sample's full loss trajectory, classifies its *shape*, and -# writes the verdict back onto the sample as the categorical tag "loss_shape". -# -# The six shapes describe how a sample's loss evolved over training: -# monotonic -> loss steadily decreasing (model is learning it) -# plateaued -> dropped then leveled off still-high (stuck / hard sample) -# Flat_high -> never moved, stayed high (mislabel / unlearnable) -# high_variance -> noisy oscillation (ambiguous label) -# U_Shape -> learned then forgotten (catastrophic interference) -# Spiked -> sudden jump at some step (data/aug/version change) - -# Allowed values for the categorical tag, in display order. -LOSS_SHAPE_LABELS = [ - "monotonic", "plateaued", "Flat_high", - "high_variance", "U_Shape", "Spiked", -] - -# Numeric encoding returned by the signal so the verdict is also plottable -# per-sample (a tag is a string; a signal must be numeric). -LOSS_SHAPE_CODES = {label: i for i, label in enumerate(LOSS_SHAPE_LABELS)} - -# Minimum number of logged points before a trajectory can be classified. -_MIN_HISTORY = 5 - -# Get checkpoint manager -checkpoint_manager = ledgers.get_checkpoint_manager() - -# Declare the categorical tag written by the loss-shape classifier signal, -# so the UI shows all six choices even before any sample is tagged. Must be -# called after the dataloader is registered (the dataframe now exists). -wl.register_categorical_tag("loss_shape", LOSS_SHAPE_LABELS) - - -def _classify_loss_shape(values): - """Classify a per-sample loss trajectory into one of LOSS_SHAPE_LABELS. - - *values* is the loss series ordered by step. Returns a label string, or - ``None`` when there is not enough history yet to decide. All thresholds - are scale-invariant (expressed as fractions of the trajectory's own - range), so the same logic works regardless of the absolute loss scale. - These thresholds are illustrative — tune them for your own task. - """ - y = np.asarray(values, dtype=float) - if y.size < _MIN_HISTORY: - return None - - n = y.size - first, last = float(y[0]), float(y[-1]) - ymin, ymax = float(y.min()), float(y.max()) - rng = max(ymax - ymin, 1e-8) - mean = float(y.mean()) - - cv = float(y.std()) / (abs(mean) + 1e-8) # noisiness (coeff. of variation) - drop = (first - last) / (abs(first) + 1e-8) # net improvement, start -> end - argmin = int(np.argmin(y)) - rebound = (last - ymin) / rng # how far it climbed back from the trough - max_up_jump = float(np.diff(y).max()) / rng # largest single-step rise - - # Flat, recent tail (last 40% of the trajectory) is used to detect plateaus. - tail = y[int(0.6 * n):] - tail_flat = (float(tail.std()) / (abs(float(tail.mean())) + 1e-8)) < 0.1 - - # Order matters: most specific / most alarming shapes are checked first. - if max_up_jump > 0.5: # one big isolated jump up - return "Spiked" - if cv > 0.5: # globally noisy / oscillating - return "high_variance" - if 0.2 * n < argmin < 0.8 * n and rebound > 0.3: # dipped mid-run, then rose - return "U_Shape" - if drop > 0.4: # large, steady net decrease - return "monotonic" - if drop > 0.15 and tail_flat: # modest drop, then leveled high - return "plateaued" - return "Flat_high" # barely moved — never learned - - -@wl.signal( - name="train_loss_sample_shape_classifier", - subscribe_to="train_loss/sample", - compute_every_n_steps=25, - log=False, # side-effecting signal: we tag samples, no aggregate curve needed -) -def classify_loss_shape(ctx: wl.SignalContext) -> int: - """Dynamic signal fired per sample every 25 steps for "train_loss/sample". - - Pulls the sample's full loss history, classifies its shape, and tags the - sample with the categorical tag ``loss_shape``. Returns the numeric shape - code (or -1 when there is not enough history yet) so the verdict is also - available as a per-sample signal column. - """ - # Full per-sample trajectory of the subscribed metric, ordered by step. - history = wl.query_sample_history(ctx.sample_id, signal_name="train_loss/sample", exp_hash=checkpoint_manager.get_current_experiment_hash()) - series = sorted(((step, val) for _, step, val, _ in history), key=lambda t: t[0]) - values = [v for _, v in series] - - label = _classify_loss_shape(values) - if label is None: - return -1 - - # Persist the verdict as a categorical tag on this sample. - wl.set_categorical_tag([ctx.sample_id], "loss_shape", label) - return LOSS_SHAPE_CODES[label] diff --git a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/utils/data.py b/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/utils/data.py deleted file mode 100644 index dbff4a02..00000000 --- a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/utils/data.py +++ /dev/null @@ -1,226 +0,0 @@ -import os -import ssl -import zipfile -import urllib.request - -import numpy as np -import torch - -from torchvision import transforms - -from torch.utils.data import Dataset - -from PIL import Image - - -# ============================================================================= -# Penn-Fudan Pedestrian detection dataset -# ============================================================================= -# A small, real object-detection dataset (~170 photos, one class: "person"). -# It ships per-instance segmentation masks; we derive an axis-aligned bounding -# box per pedestrian from each mask. Downloaded + extracted on first use. -# -# On-disk layout after extraction: -# /PennFudanPed/ -# PNGImages/FudanPed00001.png ... -# PedMasks/FudanPed00001_mask.png ... # pixel value k = k-th pedestrian, 0 = bg -# -# WL renders detection targets/predictions from a per-sample [N, 6] array -# ``[x1, y1, x2, y2, class_id, confidence]`` normalized to [0, 1] (GT conf = 1.0) -# — see ``get_items`` below and ``data_service.py`` (task_type == "detection"). - -CLASS_NAMES = ["person"] - -_URL = "https://www.cis.upenn.edu/~jshi/ped_html/PennFudanPed.zip" - -# ImageNet statistics — the MobileNetV3 backbone is pretrained with these, so we -# normalize model inputs the same way. (The UI still shows the original PNG.) -IMAGENET_MEAN = [0.485, 0.456, 0.406] -IMAGENET_STD = [0.229, 0.224, 0.225] - - -def download_penn_fudan(root): - """Download + extract Penn-Fudan into /PennFudanPed (idempotent).""" - base = os.path.join(root, "PennFudanPed") - if os.path.isdir(os.path.join(base, "PNGImages")): - return base - - os.makedirs(root, exist_ok=True) - zip_path = os.path.join(root, "PennFudanPed.zip") - - if not os.path.exists(zip_path): - print(f"[data] Downloading Penn-Fudan dataset to {zip_path} ...", flush=True) - try: - urllib.request.urlretrieve(_URL, zip_path) - except Exception as e: - # Some corporate environments break TLS verification - retry unverified. - print(f"[data] TLS verification failed ({e}); retrying without verification.", flush=True) - ctx = ssl._create_unverified_context() - with urllib.request.urlopen(_URL, context=ctx) as resp, open(zip_path, "wb") as fh: - fh.write(resp.read()) - - print("[data] Extracting Penn-Fudan ...", flush=True) - with zipfile.ZipFile(zip_path) as zf: - zf.extractall(root) - return base - - -def _boxes_from_mask(mask_path): - """Derive one bbox per pedestrian from a Penn-Fudan instance mask. - - Returns (boxes_px [N, 4] int xyxy, height, width). Background (0) skipped. - """ - mask = np.array(Image.open(mask_path)) - h, w = mask.shape[:2] - obj_ids = np.unique(mask) - obj_ids = obj_ids[obj_ids != 0] # drop background - - boxes = [] - for oid in obj_ids: - ys, xs = np.where(mask == oid) - if xs.size == 0: - continue - x1, x2 = int(xs.min()), int(xs.max()) - y1, y2 = int(ys.min()), int(ys.max()) - if x2 > x1 and y2 > y1: - boxes.append([x1, y1, x2, y2]) - return np.asarray(boxes, dtype=np.float32).reshape(-1, 4), h, w - - -class PennFudanDetectionDataset(Dataset): - """Pedestrian bounding-box detection over the Penn-Fudan images. - - Args: - root: directory to download/extract the dataset into. - split: "train" or "val" (deterministic split of the 170 images). - image_size: square resize fed to the model. - val_fraction: fraction of images held out for validation. - max_samples: optional cap on the split size (for quick runs). - """ - - def __init__( - self, - root, - split="train", - num_classes=1, - image_size=256, - val_fraction=0.2, - max_samples=None, - ): - super().__init__() - self.root = root - self.split = split - self.num_classes = num_classes - self.image_size = image_size - # Explicit task type; bypasses WL's label-shape heuristic so bboxes are - # rendered as detection overlays (not mistaken for classification). - self.task_type = "detection" - self.class_names = CLASS_NAMES[:num_classes] - - base = download_penn_fudan(root) - img_dir = os.path.join(base, "PNGImages") - mask_dir = os.path.join(base, "PedMasks") - - all_imgs = sorted(f for f in os.listdir(img_dir) if f.lower().endswith(".png")) - - # Deterministic train/val split: every k-th image goes to val. - k = max(2, int(round(1.0 / max(val_fraction, 1e-6)))) - if split == "val": - selected = all_imgs[::k] - else: - val_set = set(all_imgs[::k]) - selected = [f for f in all_imgs if f not in val_set] - - selected = selected[:max_samples] if max_samples != None else selected - - self.images = [] - self.masks = [] - for fname in selected: - base_name, _ = os.path.splitext(fname) - mask_path = os.path.join(mask_dir, base_name + "_mask.png") - if os.path.exists(mask_path): - self.images.append(os.path.join(img_dir, fname)) - self.masks.append(mask_path) - - if len(self.images) == 0: - raise RuntimeError(f"No image/mask pairs found under {base}") - - self.image_transform = transforms.Compose( - [ - transforms.Resize((image_size, image_size), interpolation=Image.BILINEAR), - transforms.ToTensor(), - transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD), - ] - ) - - def __len__(self): - return len(self.images) - - def _load_boxes(self, mask_path): - """Read mask → [N, 6] float32 = [x1, y1, x2, y2, cls=0, conf=1.0] (normalized).""" - boxes_px, h, w = _boxes_from_mask(mask_path) - if boxes_px.shape[0] == 0: - return np.zeros((0, 6), dtype=np.float32) - norm = boxes_px.copy() - norm[:, [0, 2]] /= float(w) - norm[:, [1, 3]] /= float(h) - n = norm.shape[0] - cls = np.zeros((n, 1), dtype=np.float32) # single class: person - conf = np.ones((n, 1), dtype=np.float32) - return np.concatenate([norm, cls, conf], axis=1).astype(np.float32) - - def __getitem__(self, idx): - """Returns (item, uid, target, metadata). - - - item: normalized image tensor [C, H, W] - - uid: unique sample id (string) - - target: [N, 6] float32 = [x1, y1, x2, y2, class_id, confidence] - - metadata: dict with source paths - """ - return self.get_items(idx, include_metadata=True, include_labels=True, include_images=True) - - def get_items(self, idx, include_metadata=False, include_labels=False, include_images=False): - img_path = self.images[idx] - mask_path = self.masks[idx] - uid = os.path.splitext(os.path.basename(img_path))[0] - - metadata = { - "img_path": img_path, - "mask_path": mask_path, - } if include_metadata else None - - img_t = None - if include_images: - img = Image.open(img_path).convert("RGB") - img_t = self.image_transform(img) - - target = None - if include_labels: - target = self._load_boxes(mask_path) - - return img_t, uid, target, metadata - - -def det_collate(batch): - """Collate WL per-sample tuples for object detection. - - A sample owns a variable number of boxes, so targets cannot be stacked. We - keep them as a Python list (one [N_i, 6] tensor per sample), exactly the - layout WL's per-instance helpers expect (``targets[s]`` iterates that - sample's boxes in annotation order). - - Returns: - images: FloatTensor [B, C, H, W] - ids: list[str] of length B - targets: list[B] of [N_i, 6] float tensors ([x1, y1, x2, y2, cls, conf]) - metas: list[B] of metadata dicts - """ - images = torch.stack([b[0] for b in batch], dim=0) - ids = [b[1] for b in batch] - targets = [ - torch.as_tensor(b[2], dtype=torch.float32) - if not isinstance(b[2], torch.Tensor) else b[2].float() - for b in batch - ] - metas = [b[3] if len(b) > 3 else None for b in batch] - return images, ids, targets, metas diff --git a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/utils/model.py b/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/utils/model.py deleted file mode 100644 index ed204a66..00000000 --- a/weightslab/examples/Usecases/ws-loss_shapes_classification_per_sample/utils/model.py +++ /dev/null @@ -1,134 +0,0 @@ -# ============================================================================= -# Small single-shot grid detector on a pretrained MobileNetV3-Small backbone -# ============================================================================= -# A frozen/fine-tuned ImageNet-pretrained backbone extracts features; a tiny -# detection head turns them into an S x S grid where each cell predicts ONE box: -# (objectness, tx, ty, tw, th, class_logits...). -# -# Encoding (all coordinates normalized to the [0, 1] image frame): -# * objectness = sigmoid(t_obj) -> P(box present in this cell) -# * cx = (col + sigmoid(tx)) / S -> box center, x -# * cy = (row + sigmoid(ty)) / S -> box center, y -# * w = sigmoid(tw) -> box width (fraction of image) -# * h = sigmoid(th) -> box height (fraction of image) -# * class = softmax(class_logits) -# -# Raw forward output keeps logits (loss applies the activations); `decode` -# turns logits into xyxy boxes for metrics and UI rendering. -import torch -import torch.nn as nn - -from torchvision.models import mobilenet_v3_small, MobileNet_V3_Small_Weights - - -def decode_grid(outputs, grid_size): - """Decode raw grid logits -> per-cell xyxy boxes, objectness and class probs. - - Shared by the model (``SmallDetector.decode``) and the criterions so the - encoding lives in exactly one place. - - Args: - outputs: [B, S, S, 5 + num_classes] raw logits. - grid_size: S. - - Returns: - boxes: [B, S, S, 4] xyxy in [0, 1] - obj: [B, S, S] objectness probability - cls_probs: [B, S, S, num_classes] class probabilities - """ - B, S, _, _ = outputs.shape - device = outputs.device - - obj = torch.sigmoid(outputs[..., 0]) - tx = torch.sigmoid(outputs[..., 1]) - ty = torch.sigmoid(outputs[..., 2]) - w = torch.sigmoid(outputs[..., 3]) - h = torch.sigmoid(outputs[..., 4]) - cls_probs = torch.softmax(outputs[..., 5:], dim=-1) - - # Cell-origin grid (col=x, row=y). - cols = torch.arange(S, device=device).view(1, 1, S).expand(B, S, S) - rows = torch.arange(S, device=device).view(1, S, 1).expand(B, S, S) - - cx = (cols + tx) / S - cy = (rows + ty) / S - x1 = (cx - w / 2).clamp(0, 1) - y1 = (cy - h / 2).clamp(0, 1) - x2 = (cx + w / 2).clamp(0, 1) - y2 = (cy + h / 2).clamp(0, 1) - - boxes = torch.stack([x1, y1, x2, y2], dim=-1) - return boxes, obj, cls_probs - - -class SmallDetector(nn.Module): - def __init__( - self, - in_channels=3, - num_classes=1, - image_size=256, - grid_size=8, - pretrained=True, - freeze_backbone=True, - ): - super().__init__() - # For WeightsLab - self.task_type = "detection" - self.num_classes = num_classes - self.class_names = ["person"][:num_classes] - self.grid_size = grid_size - self.image_size = image_size - self.input_shape = (1, in_channels, image_size, image_size) - - # Channels per cell: objectness(1) + box(4) + class logits(num_classes) - self.preds_per_cell = 5 + num_classes - - # --- Pretrained backbone (ImageNet) --- - weights = MobileNet_V3_Small_Weights.IMAGENET1K_V1 if pretrained else None - backbone = mobilenet_v3_small(weights=weights) - self.backbone = backbone.features # [B, 576, H/32, W/32] - backbone_out_ch = 576 - - self.freeze_backbone = freeze_backbone - if freeze_backbone: - for p in self.backbone.parameters(): - p.requires_grad = False - - # --- Detection head (lightweight, trained from scratch) --- - self.neck = nn.Sequential( - nn.Conv2d(backbone_out_ch, 256, 3, padding=1, bias=False), - nn.BatchNorm2d(256), - nn.ReLU(inplace=True), - nn.Conv2d(256, 128, 3, padding=1, bias=False), - nn.BatchNorm2d(128), - nn.ReLU(inplace=True), - ) - self.head = nn.Conv2d(128, self.preds_per_cell, 1) - - def train(self, mode=True): - """Keep a frozen backbone in eval mode (so its BatchNorm stats stay fixed).""" - super().train(mode) - if self.freeze_backbone: - self.backbone.eval() - return self - - def forward(self, x): - """Returns raw logits [B, S, S, 5 + num_classes].""" - if self.freeze_backbone: - with torch.no_grad(): - feat = self.backbone(x) - else: - feat = self.backbone(x) - - feat = self.neck(feat) - out = self.head(feat) # [B, preds_per_cell, S', S'] - - # Resize feature grid to the configured grid_size. - if out.shape[-1] != self.grid_size or out.shape[-2] != self.grid_size: - out = nn.functional.adaptive_avg_pool2d(out, self.grid_size) - # [B, C, S, S] -> [B, S, S, C] - return out.permute(0, 2, 3, 1).contiguous() - - def decode(self, outputs): - """Decode raw logits -> per-cell xyxy boxes, objectness, class probs.""" - return decode_grid(outputs, self.grid_size) diff --git a/weightslab/examples/Usecases/ws-signals-mnist/main.py b/weightslab/examples/Usecases/ws-signals-mnist/main.py new file mode 100644 index 00000000..a2b32824 --- /dev/null +++ b/weightslab/examples/Usecases/ws-signals-mnist/main.py @@ -0,0 +1,180 @@ +"""Per-sample signals on MNIST — readable, zero save_signals, with train + eval. + +Per-step user code is just the watched loss. Everything else is a @wl.signal: + entropy from ctx.logits when the loss fires + loss_norm reactive, from the logged loss + hardness reactive, from loss + entropy + +loss_shape comes from WeightsLab's built-in classifier, applied on report write +(wl.write_dataframe(loss_shape_signal=...)) — no classifier code lives here. + +Universal loss: the watched crit runs on the test split each epoch too, so test +samples get a loss trajectory and a shape as well. + +Env: WL_STRESS_EPOCHS, WL_STRESS_OUT. +""" +import os, time, gc, json, shutil + +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import Dataset +from torchvision import datasets, transforms + +import weightslab as wl +from weightslab.components.global_monitoring import guard_training_context, guard_testing_context + +LOSS = "loss_sample" +OUT = os.environ.get("WL_STRESS_OUT", "/tmp/wl_stress") +EPOCHS = int(os.environ.get("WL_STRESS_EPOCHS", "10")) +BATCH = 64 + + +def rss_gb(): + try: + for ln in open("/proc/self/status"): + if ln.startswith("VmRSS:"): + return int(ln.split()[1]) / (1024 * 1024) + except Exception: + return -1.0 + + +class SmallCNN(nn.Module): + def __init__(self): + super().__init__() + self.input_shape = (1, 1, 28, 28) + self.net = nn.Sequential( + nn.Conv2d(1, 8, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Conv2d(8, 16, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Flatten(), nn.Linear(16 * 7 * 7, 64), nn.ReLU(), nn.Linear(64, 10)) + + def forward(self, x): + return self.net(x) + + +class MNISTIdx(Dataset): + """Yields (image, uid, label). uid namespaced by split so train/test don't + collide in the shared ledger; fast_get_label skips decode at ledger init.""" + def __init__(self, root, train, base): + self.m = datasets.MNIST(root, train=train, download=True, transform=None) + self.t = transforms.ToTensor(); self.base = base + + def __len__(self): + return len(self.m) + + def __getitem__(self, i): + img, lab = self.m[i] + return self.t(img), self.base + i, lab + + def fast_get_label(self, i): + return int(self.m.targets[i]) + + +def log(msg): + print(msg, flush=True) + + +def main(): + shutil.rmtree(OUT, ignore_errors=True) + os.makedirs(OUT + "/wl_logs", exist_ok=True) + dev = torch.device("cuda" if torch.cuda.is_available() else "cpu") + torch.manual_seed(0) + metrics = open(OUT + "/metrics.jsonl", "w") + + # vanilla baseline (plain PyTorch) for the overhead comparison + vb = SmallCNN().to(dev); vo = optim.Adam(vb.parameters(), lr=0.01); vc = nn.CrossEntropyLoss() + xb = torch.randn(BATCH, 1, 28, 28, device=dev); yb = torch.randint(0, 10, (BATCH,), device=dev) + sync = (lambda: torch.cuda.synchronize()) if dev.type == "cuda" else (lambda: None) + for _ in range(10): + vo.zero_grad(); vc(vb(xb), yb).backward(); vo.step() + sync(); t0 = time.perf_counter() + for _ in range(50): + vo.zero_grad(); vc(vb(xb), yb).backward(); vo.step() + sync(); vanilla_ms = 1000 * (time.perf_counter() - t0) / 50 + del vb, vo; gc.collect() + log(f"[run] vanilla baseline = {vanilla_ms:.2f} ms/step (dev {dev})") + + # WeightsLab setup: both splits tracked, watched loss, signals + hp = {"experiment_name": "signals-mnist", "device": str(dev), "root_log_dir": OUT + "/wl_logs", + "serving_grpc": False, "serving_cli": False, "ledger_flush_max_rows": 8192, + "ledger_enable_h5_persistence": False, "experiment_dump_to_train_steps_ratio": 10_000_000, + "data": {"train_loader": {"batch_size": BATCH, "shuffle": True}}} + wl.watch_or_edit(hp, flag="hyperparameters", defaults=hp) + train_ds = MNISTIdx(OUT + "/data", train=True, base=0) + test_ds = MNISTIdx(OUT + "/data", train=False, base=1_000_000) + model = wl.watch_or_edit(SmallCNN().to(dev), flag="model", device=dev) + opt = wl.watch_or_edit(optim.Adam(model.parameters(), lr=0.01), flag="optimizer") + loader = wl.watch_or_edit(train_ds, flag="data", loader_name="train_loader", + batch_size=BATCH, shuffle=True, is_training=True, preload_labels=True) + test_loader = wl.watch_or_edit(test_ds, flag="data", loader_name="test_loader", + batch_size=256, shuffle=False, is_training=False, preload_labels=True) + crit = wl.watch_or_edit(nn.CrossEntropyLoss(reduction="none"), + flag="loss", signal_name=LOSS, per_sample=True, log=True) + + @wl.signal(name="sig/entropy", subscribe_to=LOSS, batched=True) + def entropy(b): + p = torch.softmax(b.logits, 1) + return (-(p * (p + 1e-12).log()).sum(1)).detach().cpu().numpy() + + @wl.signal(name="sig/loss_norm", inputs=[LOSS], batched=True) + def loss_norm(b): + return b.inputs[LOSS] / (float(np.mean(b.inputs[LOSS])) + 1e-8) + + @wl.signal(name="sig/hardness", inputs=[LOSS, "sig/entropy"], batched=True) + def hardness(b): + return b.inputs[LOSS] * b.inputs["sig/entropy"] + + wl.serve(serving_grpc=False, serving_cli=False) + wl.start_training(timeout=0) + log(f"[run] tracked train={len(train_ds)} test={len(test_ds)}") + + def test_eval(): + """Universal loss: watched crit over the test split each epoch.""" + with torch.no_grad(): + for tb in test_loader: + ti, tid, tl = tb[0].to(dev), tb[1], tb[2].to(dev) + with guard_testing_context: + tlg = model(ti) + crit(tlg, tl, batch_ids=tid, preds=tlg.argmax(1, keepdim=True)) + + step_times, gstep, t_run = [], 0, time.perf_counter() + for ep in range(1, EPOCHS + 1): + ep_ms = [] + for img, ids, lab in loader: + img, lab = img.to(dev), lab.to(dev) + sync(); ts = time.perf_counter() + with guard_training_context: + opt.zero_grad() + logits = model(img) + # only per-step call: the watched loss logs loss_sample and fires + # the @wl.signal chain. No save_signals. + crit(logits, lab, batch_ids=ids, preds=logits.argmax(1, keepdim=True)).mean().backward() + opt.step() + sync(); ep_ms.append(1000 * (time.perf_counter() - ts)) + gstep += 1 + test_eval() + wl_ms = float(np.mean(ep_ms)); step_times += ep_ms + rec = {"epoch": ep, "gstep": gstep, "wl_ms": round(wl_ms, 2), "vanilla_ms": round(vanilla_ms, 2), + "rss_gb": round(rss_gb(), 2), "elapsed_s": round(time.perf_counter() - t_run, 1)} + metrics.write(json.dumps(rec) + "\n"); metrics.flush() + log(f"[run] ep {ep:3d}/{EPOCHS} | WL {wl_ms:6.2f} ms vs vanilla {vanilla_ms:.2f} " + f"= +{100*(wl_ms/vanilla_ms-1):.0f}% | RSS {rec['rss_gb']:.2f} GB | {rec['elapsed_s']:.0f}s") + + import pandas as pd + # loss_shape comes from WeightsLab's built-in classifier, run on report write. + path = wl.write_dataframe(OUT + "/report.csv", format="csv", + columns=["signals", "tags"], loss_shape_signal=LOSS) + df = pd.read_csv(path) + sc = [c for c in df.columns if c.endswith("loss_shape")][0] + med = float(np.median(step_times)) + metrics.close() + log("\n[run] ===== SUMMARY =====") + log(f"[run] vanilla {vanilla_ms:.2f} ms | WL median {med:.2f} ms/step (+{100*(med/vanilla_ms-1):.0f}%)") + log(f"[run] report {len(df)} rows | loss_shape covered {df[sc].notna().sum()}/{len(df)}") + log(f"[run] shape distribution: {df[sc].value_counts(dropna=True).to_dict()}") + log(f"[run] report -> {path}") + + +if __name__ == "__main__": + main() diff --git a/weightslab/integrations/ultralytics/README.md b/weightslab/integrations/ultralytics/README.md index 9b401a4b..ef9e464c 100644 --- a/weightslab/integrations/ultralytics/README.md +++ b/weightslab/integrations/ultralytics/README.md @@ -1,21 +1,26 @@ # WeightsLab × Ultralytics -Drop-in trainer that wires WeightsLab into UL's `YOLO.train()` — per-sample +Drop-in trainers that wire WeightsLab into UL's `YOLO.train()` — per-sample signals, live studio overlay, and ledger-driven discard control with no changes to the model or to UL's training loop. +| trainer | UL base | model | task | +|------------------------------|---------------------|------------------|----------| +| `WLAwareTrainer` | `DetectionTrainer` | `yolo*n.pt` | detect | +| `WLAwareSegmentationTrainer` | `SegmentationTrainer` | `yolo*n-seg.pt` | segment | + ## Minimal use ```python import weightslab as wl from ultralytics import YOLO -from weightslab.integrations.ultralytics import WLAwareTrainer +from weightslab.integrations.ultralytics import WLAwareTrainer # or WLAwareSegmentationTrainer wl.watch_or_edit(cfg, flag="hyperparameters", defaults=cfg) wl.serve() -YOLO("yolov8n.pt").train( - trainer=WLAwareTrainer, +YOLO("yolo11n.pt").train( # yolo11n-seg.pt for segmentation + trainer=WLAwareTrainer, # WLAwareSegmentationTrainer for segmentation data="my_dataset.yaml", imgsz=640, epochs=100, batch=16, project="./logs", name="exp", workers=0, amp=False, @@ -23,6 +28,19 @@ YOLO("yolov8n.pt").train( wl.keep_serving() ``` +## Segmentation + +`WLAwareSegmentationTrainer` reuses the detection signal pack: `v8SegmentationLoss` +routes through the same `get_assigned_targets_and_loss` sync point, `Segment` +subclasses `Detect`, and `SegmentationValidator._process_batch(preds, batch)` +matches the val-IoU tap — so per-sample **box/cls/dfl** losses and **box-IoU** +flow unchanged, plus aggregate `train/seg` and mask `(M)` val metrics. Masks are +trained on (they ride through the collate into UL's loss) but tracked/rendered at +the **bbox** level in Studio for now (`task_type="detection"`); per-sample mask +signals + mask overlays are a future addition. Any segmentation-only overlay +incompatibility is absorbed by `_ship_round`'s best-effort per-signal guards, so +it never crashes training. + ## Required train kwargs | kwarg | value | why | @@ -42,7 +60,7 @@ which becomes the WL logger's `log_dir/name`). | component | tested | notes | |-------------|---------------------|------------------------------------------------| -| Ultralytics | 8.3.x | DetectionTrainer subclass; YOLOv8/v11 detect | +| Ultralytics | 8.3.x–8.4.x | Detection + Segmentation trainers; YOLOv8/v11 | | Python | 3.11 | | | Linux | ✅ | CPU + CUDA | | Windows | ⚠️ caveats | install `torchvision` CUDA wheels (else `torchvision::nms` CUDA backend missing); `dill` cannot pickle some module graphs — set `dump_model_architecture: false` in `config.yaml` | @@ -53,8 +71,10 @@ which becomes the WL logger's `log_dir/name`). - **per-sample TRAIN signals:** `train/box_per_sample`, `train/cls_per_sample`, `train/dfl_per_sample`, live NMS predictions overlay - **per-sample VAL signals:** `val/iou_per_sample`, post-NMS predictions overlay -- **aggregate train losses:** `train/{box,cls,dfl}` (from `trainer.loss_items`) +- **aggregate train losses:** `train/{box,cls,dfl}` (+ `train/seg` for + segmentation) from `trainer.loss_items` - **aggregate val metrics:** `val/{precision,recall,mAP50,mAP50-95,fitness}` + (+ mask `_mask` variants for segmentation) ## Behavior under discard diff --git a/weightslab/integrations/ultralytics/__init__.py b/weightslab/integrations/ultralytics/__init__.py index 15f7c57e..3ec79271 100644 --- a/weightslab/integrations/ultralytics/__init__.py +++ b/weightslab/integrations/ultralytics/__init__.py @@ -1,13 +1,16 @@ """WeightsLab ↔ Ultralytics integration. -Two-name public surface: +Public surface: * `WLAwareTrainer` — UL DetectionTrainer subclass that wires WL through the canonical pattern (both train/val loaders go through `wl.watch_or_edit(flag='data')`, per-sample TRAIN + VAL signals installed, live train + val NMS prediction overlays shipped to studio). - * `WLAwareDataset` — UL YOLODataset subclass that returns the WL - preview-protocol 4-tuple and provides a PIL fallback in - `fast_get_label` for ledger-init. + * `WLAwareSegmentationTrainer` — the same wiring on UL's SegmentationTrainer + for `*-seg.pt` models (masks train through the collate; per-sample signals + are box-level today — see trainer.py). + * `WLAwareDataset` / `WLAwareSegmentationDataset` — UL YOLODataset + subclasses that return the WL preview-protocol 4-tuple and provide a PIL + fallback in `fast_get_label` for ledger-init. Minimal user surface: @@ -19,7 +22,7 @@ wl.serve() YOLO(cfg["model"]).train( - trainer=WLAwareTrainer, + trainer=WLAwareTrainer, # or WLAwareSegmentationTrainer data=cfg["data_root"], imgsz=640, epochs=1000, batch=4, project="./logs", name="exp", # → WL log_dir/name workers=0, # WL invariant (parent-process uid counter) @@ -28,7 +31,12 @@ See README.md for the supported-setup matrix. """ -from .dataset import WLAwareDataset -from .trainer import WLAwareTrainer +from .dataset import WLAwareDataset, WLAwareSegmentationDataset +from .trainer import WLAwareSegmentationTrainer, WLAwareTrainer -__all__ = ["WLAwareTrainer", "WLAwareDataset"] +__all__ = [ + "WLAwareTrainer", + "WLAwareSegmentationTrainer", + "WLAwareDataset", + "WLAwareSegmentationDataset", +] diff --git a/weightslab/integrations/ultralytics/dataset.py b/weightslab/integrations/ultralytics/dataset.py index ff4be229..f3284a2a 100644 --- a/weightslab/integrations/ultralytics/dataset.py +++ b/weightslab/integrations/ultralytics/dataset.py @@ -89,3 +89,25 @@ def get_items(self, i, include_metadata=False, include_labels=False, include_ima ) labels = _to_six_col(xyxy_np, np.asarray(data["cls"])) return image, str(i), labels, metadata + + +class WLAwareSegmentationDataset(WLAwareDataset): + """YOLO segmentation dataset that speaks WL's preview protocol. + + The parent `WLAwareDataset` already extracts the per-instance bboxes that + YOLO segmentation labels carry (`lab["bboxes"]` / `data["bboxes"]`), and + the full UL sample dict — masks included — rides through + `metadata["batch"]` untouched, so `wl_ul_dict_collate` -> UL's own + `collate_fn` still assembles the `masks` tensor the segmentation loss + needs. We therefore only need to (a) keep the segmentation dataset build + flags intact (this class is assigned onto an already-built segment dataset + via ``dataset.__class__ = WLAwareSegmentationDataset``, which preserves + ``use_segments`` / ``task``), and (b) advertise the task. + + Studio tracks and renders these runs at the **bbox** level (the per-sample + signals and overlays are box-based), so `task_type` stays ``"detection"`` + — masks are trained on but not yet streamed to the grid. When per-sample + mask signals/overlays land, flip this to ``"segmentation"``. + """ + + task_type = "detection" diff --git a/weightslab/integrations/ultralytics/signals.py b/weightslab/integrations/ultralytics/signals.py index e6a733f7..c58650aa 100644 --- a/weightslab/integrations/ultralytics/signals.py +++ b/weightslab/integrations/ultralytics/signals.py @@ -23,6 +23,7 @@ """ from __future__ import annotations +import logging from dataclasses import dataclass from typing import Any, Callable, Optional @@ -41,6 +42,9 @@ from ._utils import normalize_post_nms_preds +logger = logging.getLogger(__name__) + + # ─── overlay fallback tunables ───────────────────────────────────────── # Train overlay's NMS pulls thresholds from `model.args.{conf,iou}` (the # values you'd pass through `YOLO.train(conf=..., iou=...)`). These @@ -174,19 +178,35 @@ def _ship_round(signals, channels, batch): """Iterate the signal list, ship anything the reducers produce. Wrapped in `no_grad` because signal capture is observational — running e.g. `Detect._inference` + NMS inside the train autograd graph would keep - those tensors alive until backward and compound across steps.""" + those tensors alive until backward and compound across steps. + + Signal capture is best-effort: every reducer / preds / channel call is + guarded so a signal that can't run on a given model or task (e.g. a + detection-style bbox overlay attempted on a `Segment` head whose + `_inference` output NMS can't parse) is skipped for that round instead + of crashing UL's training loop. Failures are logged at debug level.""" ids = batch["ids"] with th.no_grad(): for s in signals: - v = s.reduce(batch) + try: + v = s.reduce(batch) + except Exception as e: + logger.debug("Signal %s reduce failed; skipping round: %s", s.name, e) + continue if v is None: continue kw = {"batch_ids": ids} if s.preds is not None: - p = s.preds(batch) - if p is not None: - kw["preds"] = p - channels[s.name](v, **kw) + try: + p = s.preds(batch) + if p is not None: + kw["preds"] = p + except Exception as e: + logger.debug("Signal %s preds failed; shipping scalar only: %s", s.name, e) + try: + channels[s.name](v, **kw) + except Exception as e: + logger.debug("Signal %s channel emit failed: %s", s.name, e) def install_train_pipeline(model, signals: list[Signal]): diff --git a/weightslab/integrations/ultralytics/trainer.py b/weightslab/integrations/ultralytics/trainer.py index 6efc7aaa..2fac84cc 100644 --- a/weightslab/integrations/ultralytics/trainer.py +++ b/weightslab/integrations/ultralytics/trainer.py @@ -1,24 +1,36 @@ -"""WLAwareTrainer — UL DetectionTrainer subclass that wires WL through the -canonical pattern. +"""WL-aware UL trainers — DetectionTrainer / SegmentationTrainer subclasses +that wire WL through the canonical pattern. -What this trainer does for you: - * Builds both `train_loader` and `test_loader` through +What these trainers do for you: + * Build both `train_loader` and `test_loader` through `wl.watch_or_edit(flag='data', loader_name=...)` so each split has its own `DataSampleTrackingWrapper` with a disjoint uid range — no cross-origin collisions in the shared ledger. - * Registers the optimizer and model with WL on `on_train_start`. - * Installs per-sample TRAIN signals (cls/box/dfl per image + live NMS + * Register the optimizer and model with WL on `on_train_start`. + * Install per-sample TRAIN signals (cls/box/dfl per image + live NMS predictions overlay). - * Installs per-sample VAL signals (per-image IoU + AP@0.5 + post-NMS + * Install per-sample VAL signals (per-image IoU + AP@0.5 + post-NMS predictions overlay). - * Ships aggregate train losses + val metrics (P/R/mAP50/mAP50-95/fitness) + * Ship aggregate train losses + val metrics (P/R/mAP50/mAP50-95/fitness) as curves through WL channels. - * Wraps each train/val batch in WL's training/testing guard contexts. + * Wrap each train/val batch in WL's training/testing guard contexts. + +Two public trainers, one shared wiring mixin: + * `WLAwareTrainer` — `detect` task (YOLO*n.pt). + * `WLAwareSegmentationTrainer` — `segment` task (YOLO*n-seg.pt). Reuses the + detection signal pack: `v8SegmentationLoss` calls the same + `get_assigned_targets_and_loss` sync point, `Segment` subclasses `Detect`, + and `SegmentationValidator._process_batch(preds, batch)` matches the + val-IoU tap — so per-sample box/cls/dfl + box-IoU flow unchanged. Masks are + trained on (they ride through the collate) but tracked at the bbox level; + per-sample mask signals/overlays are a future addition. Segmentation-only + overlay quirks are absorbed by `_ship_round`'s best-effort guards. Aggregate curves shipped via UL callbacks: - * `train/{box,cls,dfl}` from `trainer.loss_items` - * `val/{precision,recall,mAP50,mAP50-95,fitness}` from - `validator.metrics.results_dict` + * `train/{box,cls,dfl}` (+ `train/seg` for segmentation) from + `trainer.loss_items` + * `val/{precision,recall,mAP50,mAP50-95,fitness}` (+ mask `(M)` variants for + segmentation) from `validator.metrics.results_dict` """ from __future__ import annotations @@ -26,29 +38,78 @@ from torch.nn import Identity from ultralytics.models.yolo.detect import DetectionTrainer +from ultralytics.models.yolo.segment import SegmentationTrainer import weightslab as wl from weightslab.backend import ledgers from .collate import wl_ul_dict_collate -from .dataset import WLAwareDataset +from .dataset import WLAwareDataset, WLAwareSegmentationDataset from .signals import install_per_sample_signals, install_per_sample_val_signals -class WLAwareTrainer(DetectionTrainer): +# ─── per-task wiring config ───────────────────────────────────────────── +# `loss_items` order per task (drives both channel names and index mapping): +# detect : (box, cls, dfl) +# segment: (box, seg, cls, dfl, sem) ← we ship the first four; sem is 0 for +# models without a semantic head. +_WL_TRAIN_LOSS_NAMES = { + "detect": ("box", "cls", "dfl"), + "segment": ("box", "seg", "cls", "dfl"), +} + +# (results_dict key -> WL channel key). Segmentation adds the mask `(M)` metrics +# alongside the box `(B)` ones. +_WL_VAL_METRICS = { + "detect": ( + ("metrics/precision(B)", "val/precision"), + ("metrics/recall(B)", "val/recall"), + ("metrics/mAP50(B)", "val/mAP50"), + ("metrics/mAP50-95(B)", "val/mAP50-95"), + ("fitness", "val/fitness"), + ), + "segment": ( + ("metrics/precision(B)", "val/precision"), + ("metrics/recall(B)", "val/recall"), + ("metrics/mAP50(B)", "val/mAP50"), + ("metrics/mAP50-95(B)", "val/mAP50-95"), + ("metrics/precision(M)", "val/precision_mask"), + ("metrics/recall(M)", "val/recall_mask"), + ("metrics/mAP50(M)", "val/mAP50_mask"), + ("metrics/mAP50-95(M)", "val/mAP50-95_mask"), + ("fitness", "val/fitness"), + ), +} + + +class _WLTrainerMixin: + """Shared WL callback wiring + dataloader construction for UL trainers. + + Concrete subclasses set two class attributes: + * `_WL_TASK` — "detect" | "segment" + * `_WL_DATASET_CLS` — the `WLAwareDataset` subclass to re-class the + UL-built dataset onto. + """ + + _WL_TASK = "detect" + _WL_DATASET_CLS = WLAwareDataset + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + task = self._WL_TASK + train_loss_names = _WL_TRAIN_LOSS_NAMES[task] + val_metrics = _WL_VAL_METRICS[task] state = {"channels": {}} def _register_channels(): ch = state["channels"] - for n in ("box", "cls", "dfl"): + for n in train_loss_names: ch[f"train/{n}"] = wl.watch_or_edit( Identity(), flag="loss", name=f"train/{n}", log=True, ) - for key in ("precision", "recall", "mAP50", "mAP50-95", "fitness"): - ch[f"val/{key}"] = wl.watch_or_edit( - Identity(), flag="metric", name=f"val/{key}", log=True, + for _ul_key, wl_key in val_metrics: + ch[wl_key] = wl.watch_or_edit( + Identity(), flag="metric", name=wl_key, log=True, ) def _on_train_start(trainer): @@ -102,8 +163,9 @@ def _on_train_batch_end(trainer): ch = state["channels"] li = getattr(trainer, "loss_items", None) if li is not None and ch: - for i, n in enumerate(("box", "cls", "dfl")): - ch[f"train/{n}"](li[i:i+1].detach()) + for i, n in enumerate(train_loss_names): + if i < li.numel(): + ch[f"train/{n}"](li[i:i+1].detach()) wl.guard_training_context.__exit__(None, None, None) def _on_val_batch_start(validator): @@ -118,13 +180,7 @@ def _on_val_end(validator): rd = getattr(md, "results_dict", None) if md is not None else None if not rd or not ch: return - for ul_key, wl_key in ( - ("metrics/precision(B)", "val/precision"), - ("metrics/recall(B)", "val/recall"), - ("metrics/mAP50(B)", "val/mAP50"), - ("metrics/mAP50-95(B)", "val/mAP50-95"), - ("fitness", "val/fitness"), - ): + for ul_key, wl_key in val_metrics: if ul_key in rd and wl_key in ch: ch[wl_key](torch.tensor([float(rd[ul_key])])) @@ -141,14 +197,14 @@ def validate(self): # so loader len reflects the active set. ({}, 0.0) unpacks cleanly into # UL's `{**self.metrics, ...}` which (None, None) does not. if len(self.test_loader) == 0: - print("[WLAwareTrainer] val skipped: all val samples are discarded", - flush=True) + print("[%s] val skipped: all val samples are discarded" + % type(self).__name__, flush=True) return {}, 0.0 return super().validate() def get_dataloader(self, dataset_path, batch_size=16, rank=0, mode="train"): dataset = self.build_dataset(dataset_path, mode, batch_size) - dataset.__class__ = WLAwareDataset + dataset.__class__ = self._WL_DATASET_CLS is_train = (mode == "train") # Get data configuration from ledger @@ -176,6 +232,26 @@ def get_dataloader(self, dataset_path, batch_size=16, rank=0, mode="train"): preload_metadata=cfg.get('preload_metadata', True), ) - if not hasattr(loader, "reset"): - loader.reset = lambda *a, **k: None + # NOTE: no `reset` shim needed here. `loader` is a ledger Proxy whose + # __getattr__ raises AttributeError for missing attrs; + # DataLoaderInterface.reset() provides the real Ultralytics-compatible + # implementation that the proxy forwards to. return loader + + +class WLAwareTrainer(_WLTrainerMixin, DetectionTrainer): + """WL-aware YOLO **detection** trainer. Drop-in for `YOLO(...).train(trainer=...)`.""" + + _WL_TASK = "detect" + _WL_DATASET_CLS = WLAwareDataset + + +class WLAwareSegmentationTrainer(_WLTrainerMixin, SegmentationTrainer): + """WL-aware YOLO **segmentation** trainer. Drop-in for `YOLO(...-seg.pt).train(trainer=...)`. + + See the module docstring: reuses the detection signal pack; masks train but + are tracked at the bbox level for now. + """ + + _WL_TASK = "segment" + _WL_DATASET_CLS = WLAwareSegmentationDataset diff --git a/weightslab/proto/experiment_service.proto b/weightslab/proto/experiment_service.proto index 40961ff4..e4d394f0 100644 --- a/weightslab/proto/experiment_service.proto +++ b/weightslab/proto/experiment_service.proto @@ -69,6 +69,7 @@ message LoggerDataPoint { message GetLatestLoggerDataResponse { repeated LoggerDataPoint points = 1; // All points from all metrics + string weightslab_version = 2; // backend weightslab package version (shown in UI logo tooltip) } message Empty {} @@ -176,6 +177,13 @@ message SaveCheckpointOperation { bool save_optimizer = 2; // also persist optimizer state } +// Restarts the backend Python process. The process relies on the container's +// restart policy (e.g. `restart: unless-stopped`) to come back up, at which +// point it reloads its last saved checkpoint the same way it does on a fresh +// container start. +message RestartInstanceOperation { +} + message TrainerCommand { bool get_hyper_parameters = 4; bool get_interactive_layers = 5; @@ -189,6 +197,7 @@ message TrainerCommand { optional DenySamplesOperation remove_eval_from_denylist_operation = 12; optional PlotNoteOperation plot_note_operation = 13; optional SaveCheckpointOperation save_checkpoint_operation = 14; + optional RestartInstanceOperation restart_operation = 15; } message HyperParameterDesc { diff --git a/weightslab/proto/experiment_service_pb2.py b/weightslab/proto/experiment_service_pb2.py index 7c109dfa..dd74b19b 100644 --- a/weightslab/proto/experiment_service_pb2.py +++ b/weightslab/proto/experiment_service_pb2.py @@ -2,7 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: weightslab/proto/experiment_service.proto -# Protobuf Python Version: 6.31.1 +# Protobuf Python Version: 5.28.1 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -11,8 +11,8 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, - 6, - 31, + 5, + 28, 1, '', 'weightslab/proto/experiment_service.proto' @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)weightslab/proto/experiment_service.proto\"\x89\x01\n\x1aGetLatestLoggerDataRequest\x12\x1c\n\x14request_full_history\x18\x01 \x01(\x08\x12\x12\n\nmax_points\x18\x02 \x01(\x05\x12\x17\n\x0f\x62reak_by_slices\x18\x03 \x01(\x08\x12\x0c\n\x04tags\x18\x04 \x03(\t\x12\x12\n\ngraph_name\x18\x05 \x01(\t\"\x81\x02\n\x0fLoggerDataPoint\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x11\n\tmodel_age\x18\x02 \x01(\x05\x12\x14\n\x0cmetric_value\x18\x03 \x01(\x02\x12\x17\n\x0f\x65xperiment_hash\x18\x04 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\x11\n\tsample_id\x18\x06 \x01(\t\x12\x1c\n\x14is_evaluation_marker\x18\x07 \x01(\x08\x12\x12\n\nsplit_name\x18\x08 \x01(\t\x12\x17\n\x0f\x65valuation_tags\x18\t \x03(\t\x12\x12\n\npoint_note\x18\n \x01(\t\x12\x12\n\naudit_mode\x18\x0b \x01(\x08\"?\n\x1bGetLatestLoggerDataResponse\x12 \n\x06points\x18\x01 \x03(\x0b\x32\x10.LoggerDataPoint\"\x07\n\x05\x45mpty\"/\n\x08NeuronId\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tneuron_id\x18\x02 \x01(\x05\"\x91\x02\n\x0fWeightOperation\x12*\n\x07op_type\x18\x01 \x01(\x0e\x32\x14.WeightOperationTypeH\x00\x88\x01\x01\x12\x15\n\x08layer_id\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1d\n\nneuron_ids\x18\x03 \x03(\x0b\x32\t.NeuronId\x12\x16\n\x0eneurons_to_add\x18\t \x01(\x05\x12 \n\x18zerofy_from_incoming_ids\x18\x0b \x03(\x05\x12\x1c\n\x14zerofy_to_neuron_ids\x18\x0c \x03(\x05\x12+\n\x11zerofy_predicates\x18\r \x03(\x0e\x32\x10.ZerofyPredicateB\n\n\x08_op_typeB\x0b\n\t_layer_id\"_\n\x17WeightsOperationRequest\x12/\n\x10weight_operation\x18\x01 \x01(\x0b\x32\x10.WeightOperationH\x00\x88\x01\x01\x42\x13\n\x11_weight_operation\"<\n\x18WeightsOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xc1\x05\n\x0fHyperParameters\x12\x1c\n\x0f\x65xperiment_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12!\n\x14training_steps_to_do\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x17\n\nbatch_size\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12 \n\x13\x66ull_eval_frequency\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12 \n\x13\x63heckpont_frequency\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bis_training\x18\x07 \x01(\x08H\x06\x88\x01\x01\x12\x15\n\x08nb_steps\x18\x08 \x01(\x05H\x07\x88\x01\x01\x12\x19\n\x0c\x61uditor_mode\x18\t \x01(\x08H\x08\x88\x01\x01\x12\x1d\n\x10train_batch_size\x18\n \x01(\x05H\t\x88\x01\x01\x12\x1b\n\x0eval_batch_size\x18\x0b \x01(\x05H\n\x88\x01\x01\x12\x1c\n\x0ftest_batch_size\x18\x0c \x01(\x05H\x0b\x88\x01\x01\x12\x1c\n\x0f\x65valuation_mode\x18\r \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11\x65valuation_config\x18\x0e \x01(\tH\r\x88\x01\x01\x42\x12\n\x10_experiment_nameB\x17\n\x15_training_steps_to_doB\x10\n\x0e_learning_rateB\r\n\x0b_batch_sizeB\x16\n\x14_full_eval_frequencyB\x16\n\x14_checkpont_frequencyB\x0e\n\x0c_is_trainingB\x0b\n\t_nb_stepsB\x0f\n\r_auditor_modeB\x13\n\x11_train_batch_sizeB\x11\n\x0f_val_batch_sizeB\x12\n\x10_test_batch_sizeB\x12\n\x10_evaluation_modeB\x14\n\x12_evaluation_config\",\n\rMetricsStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\"~\n\rAnnotatStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x08metadata\x18\x02 \x03(\x0b\x32\x1c.AnnotatStatus.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x90\x02\n\x10TrainingStatusEx\x12\x16\n\ttimestamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1c\n\x0f\x65xperiment_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tmodel_age\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12+\n\x0emetrics_status\x18\x04 \x01(\x0b\x32\x0e.MetricsStatusH\x03\x88\x01\x01\x12+\n\x0e\x61nnotat_status\x18\x05 \x01(\x0b\x32\x0e.AnnotatStatusH\x04\x88\x01\x01\x42\x0c\n\n_timestampB\x12\n\x10_experiment_nameB\x0c\n\n_model_ageB\x11\n\x0f_metrics_statusB\x11\n\x0f_annotat_status\"]\n\x15HyperParameterCommand\x12/\n\x10hyper_parameters\x18\x01 \x01(\x0b\x32\x10.HyperParametersH\x00\x88\x01\x01\x42\x13\n\x11_hyper_parameters\">\n\x14\x44\x65nySamplesOperation\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\"0\n\x17LoadCheckpointOperation\x12\x15\n\rcheckpoint_id\x18\x01 \x01(\x05\"b\n\x11PlotNoteOperation\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x11\n\tmodel_age\x18\x03 \x01(\x05\x12\x0c\n\x04note\x18\x04 \x01(\t\"L\n\x17SaveCheckpointOperation\x12\x19\n\x11save_architecture\x18\x01 \x01(\x08\x12\x16\n\x0esave_optimizer\x18\x02 \x01(\x08\"\xbc\x07\n\x0eTrainerCommand\x12\x1c\n\x14get_hyper_parameters\x18\x04 \x01(\x08\x12\x1e\n\x16get_interactive_layers\x18\x05 \x01(\x08\x12\x1d\n\x10get_data_records\x18\x06 \x01(\tH\x00\x88\x01\x01\x12%\n\x18get_single_layer_info_id\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12;\n\x16hyper_parameter_change\x18\x01 \x01(\x0b\x32\x16.HyperParameterCommandH\x02\x88\x01\x01\x12:\n\x16\x64\x65ny_samples_operation\x18\x07 \x01(\x0b\x32\x15.DenySamplesOperationH\x03\x88\x01\x01\x12?\n\x1b\x64\x65ny_eval_samples_operation\x18\n \x01(\x0b\x32\x15.DenySamplesOperationH\x04\x88\x01\x01\x12@\n\x19load_checkpoint_operation\x18\t \x01(\x0b\x32\x18.LoadCheckpointOperationH\x05\x88\x01\x01\x12\x42\n\x1eremove_from_denylist_operation\x18\x0b \x01(\x0b\x32\x15.DenySamplesOperationH\x06\x88\x01\x01\x12G\n#remove_eval_from_denylist_operation\x18\x0c \x01(\x0b\x32\x15.DenySamplesOperationH\x07\x88\x01\x01\x12\x34\n\x13plot_note_operation\x18\r \x01(\x0b\x32\x12.PlotNoteOperationH\x08\x88\x01\x01\x12@\n\x19save_checkpoint_operation\x18\x0e \x01(\x0b\x32\x18.SaveCheckpointOperationH\t\x88\x01\x01\x42\x13\n\x11_get_data_recordsB\x1b\n\x19_get_single_layer_info_idB\x19\n\x17_hyper_parameter_changeB\x19\n\x17_deny_samples_operationB\x1e\n\x1c_deny_eval_samples_operationB\x1c\n\x1a_load_checkpoint_operationB!\n\x1f_remove_from_denylist_operationB&\n$_remove_eval_from_denylist_operationB\x16\n\x14_plot_note_operationB\x1c\n\x1a_save_checkpoint_operation\"\x9d\x01\n\x12HyperParameterDesc\x12\r\n\x05label\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x1c\n\x0fnumerical_value\x18\x04 \x01(\x02H\x00\x88\x01\x01\x12\x19\n\x0cstring_value\x18\x05 \x01(\tH\x01\x88\x01\x01\x42\x12\n\x10_numerical_valueB\x0f\n\r_string_value\"\xf2\x02\n\x10NeuronStatistics\x12!\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronIdH\x00\x88\x01\x01\x12\x17\n\nneuron_age\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1f\n\x12train_trigger_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x1e\n\x11\x65val_trigger_rate\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x07 \x01(\x02H\x04\x88\x01\x01\x12\x36\n\x0bincoming_lr\x18\x08 \x03(\x0b\x32!.NeuronStatistics.IncomingLrEntry\x1a\x31\n\x0fIncomingLrEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\n_neuron_idB\r\n\x0b_neuron_ageB\x15\n\x13_train_trigger_rateB\x14\n\x12_eval_trigger_rateB\x10\n\x0e_learning_rate\"\xf0\x02\n\x13LayerRepresentation\x12\x15\n\x08layer_id\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rneurons_count\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12#\n\x16incoming_neurons_count\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x13\n\x06stride\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12-\n\x12neurons_statistics\x18\n \x03(\x0b\x32\x11.NeuronStatisticsB\x0b\n\t_layer_idB\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x10\n\x0e_neurons_countB\x19\n\x17_incoming_neurons_countB\x0e\n\x0c_kernel_sizeB\t\n\x07_stride\"H\n\x11\x41\x63tivationRequest\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tsample_id\x18\x02 \x01(\t\x12\x0e\n\x06origin\x18\x03 \x01(\t\"H\n\rActivationMap\x12\x11\n\tneuron_id\x18\x01 \x01(\x05\x12\x0e\n\x06values\x18\x02 \x03(\x02\x12\t\n\x01H\x18\x03 \x01(\x05\x12\t\n\x01W\x18\x04 \x01(\x05\"d\n\x12\x41\x63tivationResponse\x12\x12\n\nlayer_type\x18\x01 \x01(\t\x12\x15\n\rneurons_count\x18\x02 \x01(\x05\x12#\n\x0b\x61\x63tivations\x18\x03 \x03(\x0b\x32\x0e.ActivationMap\"\x93\x01\n\tTaskField\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x02H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x05H\x00\x12\x16\n\x0cstring_value\x18\x04 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_value\x18\x05 \x01(\x0cH\x00\x12\x14\n\nbool_value\x18\x06 \x01(\x08H\x00\x42\x07\n\x05value\"\xcc\x02\n\x0eRecordMetadata\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x14\n\x0csample_label\x18\x02 \x03(\x05\x12\x19\n\x11sample_prediction\x18\x03 \x03(\x05\x12=\n\x10sample_last_loss\x18\x04 \x03(\x0b\x32#.RecordMetadata.SampleLastLossEntry\x12\x19\n\x11sample_encounters\x18\x05 \x01(\x05\x12\x18\n\x10sample_discarded\x18\x06 \x01(\x08\x12 \n\x0c\x65xtra_fields\x18\x07 \x03(\x0b\x32\n.TaskField\x12\x16\n\x0eprediction_raw\x18\t \x01(\x0c\x12\x11\n\ttask_type\x18\n \x01(\t\x1a\x35\n\x13SampleLastLossEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x93\x01\n\x10SampleStatistics\x12\x13\n\x06origin\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0csample_count\x18\x07 \x01(\x05H\x01\x88\x01\x01\x12\x11\n\ttask_type\x18\t \x01(\t\x12 \n\x07records\x18\x08 \x03(\x0b\x32\x0f.RecordMetadataB\t\n\x07_originB\x0f\n\r_sample_count\"\xe6\x01\n\x0f\x43ommandResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x33\n\x16hyper_parameters_descs\x18\x03 \x03(\x0b\x32\x13.HyperParameterDesc\x12\x33\n\x15layer_representations\x18\x04 \x03(\x0b\x32\x14.LayerRepresentation\x12\x31\n\x11sample_statistics\x18\x05 \x01(\x0b\x32\x11.SampleStatisticsH\x00\x88\x01\x01\x42\x14\n\x12_sample_statistics\"U\n\rSampleRequest\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_origin\"\xad\x02\n\x15SampleRequestResponse\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x12\n\x05label\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04\x64\x61ta\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x1a\n\rerror_message\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x15\n\x08raw_data\x18\x06 \x01(\x0cH\x05\x88\x01\x01\x12\x11\n\x04mask\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x12\x17\n\nprediction\x18\x08 \x01(\x0cH\x07\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_originB\x08\n\x06_labelB\x07\n\x05_dataB\x10\n\x0e_error_messageB\x0b\n\t_raw_dataB\x07\n\x05_maskB\r\n\x0b_prediction\"\x92\x01\n\x12\x42\x61tchSampleRequest\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x19\n\x0cresize_width\x18\x03 \x01(\x05H\x00\x88\x01\x01\x12\x1a\n\rresize_height\x18\x04 \x01(\x05H\x01\x88\x01\x01\x42\x0f\n\r_resize_widthB\x10\n\x0e_resize_height\">\n\x13\x42\x61tchSampleResponse\x12\'\n\x07samples\x18\x01 \x03(\x0b\x32\x16.SampleRequestResponse\".\n\x0eWeightsRequest\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\"\x9d\x02\n\x0fWeightsResponse\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08incoming\x18\x04 \x01(\x05\x12\x10\n\x08outgoing\x18\x05 \x01(\x05\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x02\x88\x01\x01\x12\x0f\n\x07weights\x18\x07 \x03(\x02\x12\x0f\n\x07success\x18\x0b \x01(\x08\x12\x1a\n\rerror_message\x18\x0c \x01(\tH\x03\x88\x01\x01\x42\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x0e\n\x0c_kernel_sizeB\x10\n\x0e_error_message\"R\n\x10\x44\x61taQueryRequest\x12\r\n\x05query\x18\x01 \x01(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\x12\x1b\n\x13is_natural_language\x18\x03 \x01(\x08\"5\n\x11\x43\x61tegoricalTagDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ncategories\x18\x02 \x03(\t\"\xa9\x02\n\x11\x44\x61taQueryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1d\n\x15number_of_all_samples\x18\x03 \x01(\x05\x12%\n\x1dnumber_of_samples_in_the_loop\x18\x04 \x01(\x05\x12#\n\x1bnumber_of_discarded_samples\x18\x05 \x01(\x05\x12\x13\n\x0bunique_tags\x18\x06 \x03(\t\x12+\n\x11\x61gent_intent_type\x18\x07 \x01(\x0e\x32\x10.AgentIntentType\x12\x17\n\x0f\x61nalysis_result\x18\x08 \x01(\t\x12,\n\x10\x63\x61tegorical_tags\x18\t \x03(\x0b\x32\x12.CategoricalTagDef\"\xc2\x01\n\x12\x44\x61taSamplesRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12 \n\x18include_transformed_data\x18\x03 \x01(\x08\x12\x18\n\x10include_raw_data\x18\x04 \x01(\x08\x12\x19\n\x11stats_to_retrieve\x18\x05 \x03(\t\x12\x14\n\x0cresize_width\x18\x06 \x01(\x05\x12\x15\n\rresize_height\x18\x07 \x01(\x05\"m\n\x08\x44\x61taStat\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x05\x12\r\n\x05value\x18\x04 \x03(\x02\x12\x14\n\x0cvalue_string\x18\x05 \x01(\t\x12\x11\n\tthumbnail\x18\x06 \x01(\x0c\">\n\nDataRecord\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x1d\n\ndata_stats\x18\x02 \x03(\x0b\x32\t.DataStat\"Z\n\x13\x44\x61taSamplesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12!\n\x0c\x64\x61ta_records\x18\x03 \x03(\x0b\x32\x0b.DataRecord\"C\n\x0fHistogramSubBar\x12\x0e\n\x06origin\x18\x01 \x01(\t\x12\x11\n\tdiscarded\x18\x02 \x01(\x08\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\"h\n\x0cHistogramBin\x12\x0b\n\x03min\x18\x01 \x01(\x01\x12\x0b\n\x03max\x18\x02 \x01(\x01\x12\x0b\n\x03\x61vg\x18\x03 \x01(\x01\x12\r\n\x05\x63ount\x18\x04 \x01(\x03\x12\"\n\x08sub_bars\x18\x05 \x03(\x0b\x32\x10.HistogramSubBar\"[\n\x17\x43\x61tegoricalHistogramBar\x12\r\n\x05label\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x03\x12\"\n\x08sub_bars\x18\x03 \x03(\x0b\x32\x10.HistogramSubBar\"4\n\x10HistogramRequest\x12\x0e\n\x06\x63olumn\x18\x01 \x01(\t\x12\x10\n\x08max_bins\x18\x02 \x01(\x05\"\xb2\x01\n\x11HistogramResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\ntotal_rows\x18\x03 \x01(\x03\x12\x1b\n\x04\x62ins\x18\x04 \x03(\x0b\x32\r.HistogramBin\x12\x16\n\x0eis_categorical\x18\x05 \x01(\x08\x12\x32\n\x10\x63\x61tegorical_bars\x18\x06 \x03(\x0b\x32\x18.CategoricalHistogramBar\"W\n\x12GetMetaDataRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12\x17\n\x0fmodal_sample_id\x18\x03 \x01(\t\"\x99\x01\n\x13GetMetaDataResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1a\n\x12\x61ll_metadata_names\x18\x03 \x03(\t\x12!\n\x0cgrid_records\x18\x04 \x03(\x0b\x32\x0b.DataRecord\x12!\n\x0cmodal_record\x18\x05 \x01(\x0b\x32\x0b.DataRecord\"J\n\x11PointCloudRequest\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\"\xbf\x01\n\x0fPointCloudChunk\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\nnum_points\x18\x03 \x01(\x05\x12\x14\n\x0cnum_features\x18\x04 \x01(\x05\x12\x10\n\x08pc_range\x18\x05 \x03(\x02\x12\x0c\n\x04\x64\x61ta\x18\x06 \x01(\x0c\x12\x13\n\x0b\x63hunk_index\x18\x07 \x01(\x05\x12\x14\n\x0ctotal_chunks\x18\x08 \x01(\x05\x12\x15\n\rfeature_names\x18\t \x03(\t\"\xdc\x01\n\x10\x44\x61taEditsRequest\x12\x11\n\tstat_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66loat_value\x18\x02 \x01(\x02\x12\x14\n\x0cstring_value\x18\x03 \x01(\t\x12\x12\n\nbool_value\x18\x04 \x01(\x08\x12\x1d\n\x04type\x18\x05 \x01(\x0e\x32\x0f.SampleEditType\x12\x13\n\x0bsamples_ids\x18\x06 \x03(\t\x12\x16\n\x0esample_origins\x18\x07 \x03(\t\x12\x16\n\x0eis_categorical\x18\x08 \x01(\x08\x12\x12\n\ncategories\x18\t \x03(\t\"5\n\x11\x44\x61taEditsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\":\n\x12\x44\x61taSplitsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x13\n\x0bsplit_names\x18\x02 \x03(\t\"9\n\x13\x41gentHealthResponse\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"^\n\x16InitializeAgentRequest\x12\x0f\n\x07\x61pi_key\x18\x01 \x01(\t\x12$\n\x08provider\x18\x02 \x01(\x0e\x32\x12.AgentProviderType\x12\r\n\x05model\x18\x03 \x01(\t\";\n\x17InitializeAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"(\n\x17\x43hangeAgentModelRequest\x12\r\n\x05model\x18\x01 \x01(\t\"<\n\x18\x43hangeAgentModelResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x17\n\x15GetAgentModelsRequest\"J\n\x16GetAgentModelsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06models\x18\x02 \x03(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"6\n\x12ResetAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"3\n\x18RestoreCheckpointRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\"=\n\x19RestoreCheckpointResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"R\n\x18TriggerEvaluationRequest\x12\x12\n\nsplit_name\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12\x14\n\x0cuse_full_set\x18\x03 \x01(\x08\"=\n\x19TriggerEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1c\n\x1aGetEvaluationStatusRequest\"\x81\x01\n\x1bGetEvaluationStatusResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x0f\n\x07\x63urrent\x18\x02 \x01(\x05\x12\r\n\x05total\x18\x03 \x01(\x05\x12\x0f\n\x07message\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12\x12\n\nsplit_name\x18\x06 \x01(\t\")\n\x17\x43\x61ncelEvaluationRequest\x12\x0e\n\x06reason\x18\x01 \x01(\t\"<\n\x18\x43\x61ncelEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t*d\n\x13WeightOperationType\x12\n\n\x06ZEROFY\x10\x00\x12\x10\n\x0cREINITIALIZE\x10\x01\x12\n\n\x06\x46REEZE\x10\x02\x12\x12\n\x0eREMOVE_NEURONS\x10\t\x12\x0f\n\x0b\x41\x44\x44_NEURONS\x10\n*o\n\x0fZerofyPredicate\x12\x19\n\x15ZEROFY_PREDICATE_NONE\x10\x00\x12 \n\x1cZEROFY_PREDICATE_WITH_FROZEN\x10\x01\x12\x1f\n\x1bZEROFY_PREDICATE_WITH_OLDER\x10\x02*M\n\x0f\x41gentIntentType\x12\x12\n\x0eINTENT_UNKNOWN\x10\x00\x12\x11\n\rINTENT_FILTER\x10\x01\x12\x13\n\x0fINTENT_ANALYSIS\x10\x02*I\n\x0eSampleEditType\x12\x11\n\rEDIT_OVERRIDE\x10\x00\x12\x13\n\x0f\x45\x44IT_ACCUMULATE\x10\x01\x12\x0f\n\x0b\x45\x44IT_REMOVE\x10\x02*,\n\x11\x41gentProviderType\x12\x17\n\x13PROVIDER_OPENROUTER\x10\x00\x32\xf5\n\n\x11\x45xperimentService\x12P\n\x13GetLatestLoggerData\x12\x1b.GetLatestLoggerDataRequest\x1a\x1c.GetLatestLoggerDataResponse\x12\x36\n\x11\x45xperimentCommand\x12\x0f.TrainerCommand\x1a\x10.CommandResponse\x12H\n\x11ManipulateWeights\x12\x18.WeightsOperationRequest\x1a\x19.WeightsOperationResponse\x12/\n\nGetWeights\x12\x0f.WeightsRequest\x1a\x10.WeightsResponse\x12\x39\n\x0eGetActivations\x12\x12.ActivationRequest\x1a\x13.ActivationResponse\x12\x37\n\nGetSamples\x12\x13.BatchSampleRequest\x1a\x14.BatchSampleResponse\x12\x37\n\x0e\x41pplyDataQuery\x12\x11.DataQueryRequest\x1a\x12.DataQueryResponse\x12;\n\x0eGetDataSamples\x12\x13.DataSamplesRequest\x1a\x14.DataSamplesResponse\x12\x35\n\x0cGetHistogram\x12\x11.HistogramRequest\x1a\x12.HistogramResponse\x12\x38\n\x0bGetMetaData\x12\x13.GetMetaDataRequest\x1a\x14.GetMetaDataResponse\x12\x37\n\rGetPointCloud\x12\x12.PointCloudRequest\x1a\x10.PointCloudChunk0\x01\x12\x37\n\x0e\x45\x64itDataSample\x12\x11.DataEditsRequest\x1a\x12.DataEditsResponse\x12,\n\rGetDataSplits\x12\x06.Empty\x1a\x13.DataSplitsResponse\x12\x30\n\x10\x43heckAgentHealth\x12\x06.Empty\x1a\x14.AgentHealthResponse\x12\x44\n\x0fInitializeAgent\x12\x17.InitializeAgentRequest\x1a\x18.InitializeAgentResponse\x12G\n\x10\x43hangeAgentModel\x12\x18.ChangeAgentModelRequest\x1a\x19.ChangeAgentModelResponse\x12\x41\n\x0eGetAgentModels\x12\x16.GetAgentModelsRequest\x1a\x17.GetAgentModelsResponse\x12)\n\nResetAgent\x12\x06.Empty\x1a\x13.ResetAgentResponse\x12J\n\x11RestoreCheckpoint\x12\x19.RestoreCheckpointRequest\x1a\x1a.RestoreCheckpointResponse\x12J\n\x11TriggerEvaluation\x12\x19.TriggerEvaluationRequest\x1a\x1a.TriggerEvaluationResponse\x12P\n\x13GetEvaluationStatus\x12\x1b.GetEvaluationStatusRequest\x1a\x1c.GetEvaluationStatusResponse\x12G\n\x10\x43\x61ncelEvaluation\x12\x18.CancelEvaluationRequest\x1a\x19.CancelEvaluationResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)weightslab/proto/experiment_service.proto\"\x89\x01\n\x1aGetLatestLoggerDataRequest\x12\x1c\n\x14request_full_history\x18\x01 \x01(\x08\x12\x12\n\nmax_points\x18\x02 \x01(\x05\x12\x17\n\x0f\x62reak_by_slices\x18\x03 \x01(\x08\x12\x0c\n\x04tags\x18\x04 \x03(\t\x12\x12\n\ngraph_name\x18\x05 \x01(\t\"\x81\x02\n\x0fLoggerDataPoint\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x11\n\tmodel_age\x18\x02 \x01(\x05\x12\x14\n\x0cmetric_value\x18\x03 \x01(\x02\x12\x17\n\x0f\x65xperiment_hash\x18\x04 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\x11\n\tsample_id\x18\x06 \x01(\t\x12\x1c\n\x14is_evaluation_marker\x18\x07 \x01(\x08\x12\x12\n\nsplit_name\x18\x08 \x01(\t\x12\x17\n\x0f\x65valuation_tags\x18\t \x03(\t\x12\x12\n\npoint_note\x18\n \x01(\t\x12\x12\n\naudit_mode\x18\x0b \x01(\x08\"[\n\x1bGetLatestLoggerDataResponse\x12 \n\x06points\x18\x01 \x03(\x0b\x32\x10.LoggerDataPoint\x12\x1a\n\x12weightslab_version\x18\x02 \x01(\t\"\x07\n\x05\x45mpty\"/\n\x08NeuronId\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tneuron_id\x18\x02 \x01(\x05\"\x91\x02\n\x0fWeightOperation\x12*\n\x07op_type\x18\x01 \x01(\x0e\x32\x14.WeightOperationTypeH\x00\x88\x01\x01\x12\x15\n\x08layer_id\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1d\n\nneuron_ids\x18\x03 \x03(\x0b\x32\t.NeuronId\x12\x16\n\x0eneurons_to_add\x18\t \x01(\x05\x12 \n\x18zerofy_from_incoming_ids\x18\x0b \x03(\x05\x12\x1c\n\x14zerofy_to_neuron_ids\x18\x0c \x03(\x05\x12+\n\x11zerofy_predicates\x18\r \x03(\x0e\x32\x10.ZerofyPredicateB\n\n\x08_op_typeB\x0b\n\t_layer_id\"_\n\x17WeightsOperationRequest\x12/\n\x10weight_operation\x18\x01 \x01(\x0b\x32\x10.WeightOperationH\x00\x88\x01\x01\x42\x13\n\x11_weight_operation\"<\n\x18WeightsOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xc1\x05\n\x0fHyperParameters\x12\x1c\n\x0f\x65xperiment_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12!\n\x14training_steps_to_do\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x17\n\nbatch_size\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12 \n\x13\x66ull_eval_frequency\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12 \n\x13\x63heckpont_frequency\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bis_training\x18\x07 \x01(\x08H\x06\x88\x01\x01\x12\x15\n\x08nb_steps\x18\x08 \x01(\x05H\x07\x88\x01\x01\x12\x19\n\x0c\x61uditor_mode\x18\t \x01(\x08H\x08\x88\x01\x01\x12\x1d\n\x10train_batch_size\x18\n \x01(\x05H\t\x88\x01\x01\x12\x1b\n\x0eval_batch_size\x18\x0b \x01(\x05H\n\x88\x01\x01\x12\x1c\n\x0ftest_batch_size\x18\x0c \x01(\x05H\x0b\x88\x01\x01\x12\x1c\n\x0f\x65valuation_mode\x18\r \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11\x65valuation_config\x18\x0e \x01(\tH\r\x88\x01\x01\x42\x12\n\x10_experiment_nameB\x17\n\x15_training_steps_to_doB\x10\n\x0e_learning_rateB\r\n\x0b_batch_sizeB\x16\n\x14_full_eval_frequencyB\x16\n\x14_checkpont_frequencyB\x0e\n\x0c_is_trainingB\x0b\n\t_nb_stepsB\x0f\n\r_auditor_modeB\x13\n\x11_train_batch_sizeB\x11\n\x0f_val_batch_sizeB\x12\n\x10_test_batch_sizeB\x12\n\x10_evaluation_modeB\x14\n\x12_evaluation_config\",\n\rMetricsStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\"~\n\rAnnotatStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x08metadata\x18\x02 \x03(\x0b\x32\x1c.AnnotatStatus.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x90\x02\n\x10TrainingStatusEx\x12\x16\n\ttimestamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1c\n\x0f\x65xperiment_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tmodel_age\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12+\n\x0emetrics_status\x18\x04 \x01(\x0b\x32\x0e.MetricsStatusH\x03\x88\x01\x01\x12+\n\x0e\x61nnotat_status\x18\x05 \x01(\x0b\x32\x0e.AnnotatStatusH\x04\x88\x01\x01\x42\x0c\n\n_timestampB\x12\n\x10_experiment_nameB\x0c\n\n_model_ageB\x11\n\x0f_metrics_statusB\x11\n\x0f_annotat_status\"]\n\x15HyperParameterCommand\x12/\n\x10hyper_parameters\x18\x01 \x01(\x0b\x32\x10.HyperParametersH\x00\x88\x01\x01\x42\x13\n\x11_hyper_parameters\">\n\x14\x44\x65nySamplesOperation\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\"0\n\x17LoadCheckpointOperation\x12\x15\n\rcheckpoint_id\x18\x01 \x01(\x05\"b\n\x11PlotNoteOperation\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x11\n\tmodel_age\x18\x03 \x01(\x05\x12\x0c\n\x04note\x18\x04 \x01(\t\"L\n\x17SaveCheckpointOperation\x12\x19\n\x11save_architecture\x18\x01 \x01(\x08\x12\x16\n\x0esave_optimizer\x18\x02 \x01(\x08\"\x1a\n\x18RestartInstanceOperation\"\x8d\x08\n\x0eTrainerCommand\x12\x1c\n\x14get_hyper_parameters\x18\x04 \x01(\x08\x12\x1e\n\x16get_interactive_layers\x18\x05 \x01(\x08\x12\x1d\n\x10get_data_records\x18\x06 \x01(\tH\x00\x88\x01\x01\x12%\n\x18get_single_layer_info_id\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12;\n\x16hyper_parameter_change\x18\x01 \x01(\x0b\x32\x16.HyperParameterCommandH\x02\x88\x01\x01\x12:\n\x16\x64\x65ny_samples_operation\x18\x07 \x01(\x0b\x32\x15.DenySamplesOperationH\x03\x88\x01\x01\x12?\n\x1b\x64\x65ny_eval_samples_operation\x18\n \x01(\x0b\x32\x15.DenySamplesOperationH\x04\x88\x01\x01\x12@\n\x19load_checkpoint_operation\x18\t \x01(\x0b\x32\x18.LoadCheckpointOperationH\x05\x88\x01\x01\x12\x42\n\x1eremove_from_denylist_operation\x18\x0b \x01(\x0b\x32\x15.DenySamplesOperationH\x06\x88\x01\x01\x12G\n#remove_eval_from_denylist_operation\x18\x0c \x01(\x0b\x32\x15.DenySamplesOperationH\x07\x88\x01\x01\x12\x34\n\x13plot_note_operation\x18\r \x01(\x0b\x32\x12.PlotNoteOperationH\x08\x88\x01\x01\x12@\n\x19save_checkpoint_operation\x18\x0e \x01(\x0b\x32\x18.SaveCheckpointOperationH\t\x88\x01\x01\x12\x39\n\x11restart_operation\x18\x0f \x01(\x0b\x32\x19.RestartInstanceOperationH\n\x88\x01\x01\x42\x13\n\x11_get_data_recordsB\x1b\n\x19_get_single_layer_info_idB\x19\n\x17_hyper_parameter_changeB\x19\n\x17_deny_samples_operationB\x1e\n\x1c_deny_eval_samples_operationB\x1c\n\x1a_load_checkpoint_operationB!\n\x1f_remove_from_denylist_operationB&\n$_remove_eval_from_denylist_operationB\x16\n\x14_plot_note_operationB\x1c\n\x1a_save_checkpoint_operationB\x14\n\x12_restart_operation\"\x9d\x01\n\x12HyperParameterDesc\x12\r\n\x05label\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x1c\n\x0fnumerical_value\x18\x04 \x01(\x02H\x00\x88\x01\x01\x12\x19\n\x0cstring_value\x18\x05 \x01(\tH\x01\x88\x01\x01\x42\x12\n\x10_numerical_valueB\x0f\n\r_string_value\"\xf2\x02\n\x10NeuronStatistics\x12!\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronIdH\x00\x88\x01\x01\x12\x17\n\nneuron_age\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1f\n\x12train_trigger_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x1e\n\x11\x65val_trigger_rate\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x07 \x01(\x02H\x04\x88\x01\x01\x12\x36\n\x0bincoming_lr\x18\x08 \x03(\x0b\x32!.NeuronStatistics.IncomingLrEntry\x1a\x31\n\x0fIncomingLrEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\n_neuron_idB\r\n\x0b_neuron_ageB\x15\n\x13_train_trigger_rateB\x14\n\x12_eval_trigger_rateB\x10\n\x0e_learning_rate\"\xf0\x02\n\x13LayerRepresentation\x12\x15\n\x08layer_id\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rneurons_count\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12#\n\x16incoming_neurons_count\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x13\n\x06stride\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12-\n\x12neurons_statistics\x18\n \x03(\x0b\x32\x11.NeuronStatisticsB\x0b\n\t_layer_idB\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x10\n\x0e_neurons_countB\x19\n\x17_incoming_neurons_countB\x0e\n\x0c_kernel_sizeB\t\n\x07_stride\"H\n\x11\x41\x63tivationRequest\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tsample_id\x18\x02 \x01(\t\x12\x0e\n\x06origin\x18\x03 \x01(\t\"H\n\rActivationMap\x12\x11\n\tneuron_id\x18\x01 \x01(\x05\x12\x0e\n\x06values\x18\x02 \x03(\x02\x12\t\n\x01H\x18\x03 \x01(\x05\x12\t\n\x01W\x18\x04 \x01(\x05\"d\n\x12\x41\x63tivationResponse\x12\x12\n\nlayer_type\x18\x01 \x01(\t\x12\x15\n\rneurons_count\x18\x02 \x01(\x05\x12#\n\x0b\x61\x63tivations\x18\x03 \x03(\x0b\x32\x0e.ActivationMap\"\x93\x01\n\tTaskField\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x02H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x05H\x00\x12\x16\n\x0cstring_value\x18\x04 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_value\x18\x05 \x01(\x0cH\x00\x12\x14\n\nbool_value\x18\x06 \x01(\x08H\x00\x42\x07\n\x05value\"\xcc\x02\n\x0eRecordMetadata\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x14\n\x0csample_label\x18\x02 \x03(\x05\x12\x19\n\x11sample_prediction\x18\x03 \x03(\x05\x12=\n\x10sample_last_loss\x18\x04 \x03(\x0b\x32#.RecordMetadata.SampleLastLossEntry\x12\x19\n\x11sample_encounters\x18\x05 \x01(\x05\x12\x18\n\x10sample_discarded\x18\x06 \x01(\x08\x12 \n\x0c\x65xtra_fields\x18\x07 \x03(\x0b\x32\n.TaskField\x12\x16\n\x0eprediction_raw\x18\t \x01(\x0c\x12\x11\n\ttask_type\x18\n \x01(\t\x1a\x35\n\x13SampleLastLossEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x93\x01\n\x10SampleStatistics\x12\x13\n\x06origin\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0csample_count\x18\x07 \x01(\x05H\x01\x88\x01\x01\x12\x11\n\ttask_type\x18\t \x01(\t\x12 \n\x07records\x18\x08 \x03(\x0b\x32\x0f.RecordMetadataB\t\n\x07_originB\x0f\n\r_sample_count\"\xe6\x01\n\x0f\x43ommandResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x33\n\x16hyper_parameters_descs\x18\x03 \x03(\x0b\x32\x13.HyperParameterDesc\x12\x33\n\x15layer_representations\x18\x04 \x03(\x0b\x32\x14.LayerRepresentation\x12\x31\n\x11sample_statistics\x18\x05 \x01(\x0b\x32\x11.SampleStatisticsH\x00\x88\x01\x01\x42\x14\n\x12_sample_statistics\"U\n\rSampleRequest\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_origin\"\xad\x02\n\x15SampleRequestResponse\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x12\n\x05label\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04\x64\x61ta\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x1a\n\rerror_message\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x15\n\x08raw_data\x18\x06 \x01(\x0cH\x05\x88\x01\x01\x12\x11\n\x04mask\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x12\x17\n\nprediction\x18\x08 \x01(\x0cH\x07\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_originB\x08\n\x06_labelB\x07\n\x05_dataB\x10\n\x0e_error_messageB\x0b\n\t_raw_dataB\x07\n\x05_maskB\r\n\x0b_prediction\"\x92\x01\n\x12\x42\x61tchSampleRequest\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x19\n\x0cresize_width\x18\x03 \x01(\x05H\x00\x88\x01\x01\x12\x1a\n\rresize_height\x18\x04 \x01(\x05H\x01\x88\x01\x01\x42\x0f\n\r_resize_widthB\x10\n\x0e_resize_height\">\n\x13\x42\x61tchSampleResponse\x12\'\n\x07samples\x18\x01 \x03(\x0b\x32\x16.SampleRequestResponse\".\n\x0eWeightsRequest\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\"\x9d\x02\n\x0fWeightsResponse\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08incoming\x18\x04 \x01(\x05\x12\x10\n\x08outgoing\x18\x05 \x01(\x05\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x02\x88\x01\x01\x12\x0f\n\x07weights\x18\x07 \x03(\x02\x12\x0f\n\x07success\x18\x0b \x01(\x08\x12\x1a\n\rerror_message\x18\x0c \x01(\tH\x03\x88\x01\x01\x42\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x0e\n\x0c_kernel_sizeB\x10\n\x0e_error_message\"R\n\x10\x44\x61taQueryRequest\x12\r\n\x05query\x18\x01 \x01(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\x12\x1b\n\x13is_natural_language\x18\x03 \x01(\x08\"5\n\x11\x43\x61tegoricalTagDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ncategories\x18\x02 \x03(\t\"\xa9\x02\n\x11\x44\x61taQueryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1d\n\x15number_of_all_samples\x18\x03 \x01(\x05\x12%\n\x1dnumber_of_samples_in_the_loop\x18\x04 \x01(\x05\x12#\n\x1bnumber_of_discarded_samples\x18\x05 \x01(\x05\x12\x13\n\x0bunique_tags\x18\x06 \x03(\t\x12+\n\x11\x61gent_intent_type\x18\x07 \x01(\x0e\x32\x10.AgentIntentType\x12\x17\n\x0f\x61nalysis_result\x18\x08 \x01(\t\x12,\n\x10\x63\x61tegorical_tags\x18\t \x03(\x0b\x32\x12.CategoricalTagDef\"\xc2\x01\n\x12\x44\x61taSamplesRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12 \n\x18include_transformed_data\x18\x03 \x01(\x08\x12\x18\n\x10include_raw_data\x18\x04 \x01(\x08\x12\x19\n\x11stats_to_retrieve\x18\x05 \x03(\t\x12\x14\n\x0cresize_width\x18\x06 \x01(\x05\x12\x15\n\rresize_height\x18\x07 \x01(\x05\"m\n\x08\x44\x61taStat\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x05\x12\r\n\x05value\x18\x04 \x03(\x02\x12\x14\n\x0cvalue_string\x18\x05 \x01(\t\x12\x11\n\tthumbnail\x18\x06 \x01(\x0c\">\n\nDataRecord\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x1d\n\ndata_stats\x18\x02 \x03(\x0b\x32\t.DataStat\"Z\n\x13\x44\x61taSamplesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12!\n\x0c\x64\x61ta_records\x18\x03 \x03(\x0b\x32\x0b.DataRecord\"C\n\x0fHistogramSubBar\x12\x0e\n\x06origin\x18\x01 \x01(\t\x12\x11\n\tdiscarded\x18\x02 \x01(\x08\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\"h\n\x0cHistogramBin\x12\x0b\n\x03min\x18\x01 \x01(\x01\x12\x0b\n\x03max\x18\x02 \x01(\x01\x12\x0b\n\x03\x61vg\x18\x03 \x01(\x01\x12\r\n\x05\x63ount\x18\x04 \x01(\x03\x12\"\n\x08sub_bars\x18\x05 \x03(\x0b\x32\x10.HistogramSubBar\"[\n\x17\x43\x61tegoricalHistogramBar\x12\r\n\x05label\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x03\x12\"\n\x08sub_bars\x18\x03 \x03(\x0b\x32\x10.HistogramSubBar\"4\n\x10HistogramRequest\x12\x0e\n\x06\x63olumn\x18\x01 \x01(\t\x12\x10\n\x08max_bins\x18\x02 \x01(\x05\"\xb2\x01\n\x11HistogramResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\ntotal_rows\x18\x03 \x01(\x03\x12\x1b\n\x04\x62ins\x18\x04 \x03(\x0b\x32\r.HistogramBin\x12\x16\n\x0eis_categorical\x18\x05 \x01(\x08\x12\x32\n\x10\x63\x61tegorical_bars\x18\x06 \x03(\x0b\x32\x18.CategoricalHistogramBar\"W\n\x12GetMetaDataRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12\x17\n\x0fmodal_sample_id\x18\x03 \x01(\t\"\x99\x01\n\x13GetMetaDataResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1a\n\x12\x61ll_metadata_names\x18\x03 \x03(\t\x12!\n\x0cgrid_records\x18\x04 \x03(\x0b\x32\x0b.DataRecord\x12!\n\x0cmodal_record\x18\x05 \x01(\x0b\x32\x0b.DataRecord\"J\n\x11PointCloudRequest\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\"\xbf\x01\n\x0fPointCloudChunk\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\nnum_points\x18\x03 \x01(\x05\x12\x14\n\x0cnum_features\x18\x04 \x01(\x05\x12\x10\n\x08pc_range\x18\x05 \x03(\x02\x12\x0c\n\x04\x64\x61ta\x18\x06 \x01(\x0c\x12\x13\n\x0b\x63hunk_index\x18\x07 \x01(\x05\x12\x14\n\x0ctotal_chunks\x18\x08 \x01(\x05\x12\x15\n\rfeature_names\x18\t \x03(\t\"\xdc\x01\n\x10\x44\x61taEditsRequest\x12\x11\n\tstat_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66loat_value\x18\x02 \x01(\x02\x12\x14\n\x0cstring_value\x18\x03 \x01(\t\x12\x12\n\nbool_value\x18\x04 \x01(\x08\x12\x1d\n\x04type\x18\x05 \x01(\x0e\x32\x0f.SampleEditType\x12\x13\n\x0bsamples_ids\x18\x06 \x03(\t\x12\x16\n\x0esample_origins\x18\x07 \x03(\t\x12\x16\n\x0eis_categorical\x18\x08 \x01(\x08\x12\x12\n\ncategories\x18\t \x03(\t\"5\n\x11\x44\x61taEditsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\":\n\x12\x44\x61taSplitsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x13\n\x0bsplit_names\x18\x02 \x03(\t\"9\n\x13\x41gentHealthResponse\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"^\n\x16InitializeAgentRequest\x12\x0f\n\x07\x61pi_key\x18\x01 \x01(\t\x12$\n\x08provider\x18\x02 \x01(\x0e\x32\x12.AgentProviderType\x12\r\n\x05model\x18\x03 \x01(\t\";\n\x17InitializeAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"(\n\x17\x43hangeAgentModelRequest\x12\r\n\x05model\x18\x01 \x01(\t\"<\n\x18\x43hangeAgentModelResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x17\n\x15GetAgentModelsRequest\"J\n\x16GetAgentModelsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06models\x18\x02 \x03(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"6\n\x12ResetAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"3\n\x18RestoreCheckpointRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\"=\n\x19RestoreCheckpointResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"R\n\x18TriggerEvaluationRequest\x12\x12\n\nsplit_name\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12\x14\n\x0cuse_full_set\x18\x03 \x01(\x08\"=\n\x19TriggerEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1c\n\x1aGetEvaluationStatusRequest\"\x81\x01\n\x1bGetEvaluationStatusResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x0f\n\x07\x63urrent\x18\x02 \x01(\x05\x12\r\n\x05total\x18\x03 \x01(\x05\x12\x0f\n\x07message\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12\x12\n\nsplit_name\x18\x06 \x01(\t\")\n\x17\x43\x61ncelEvaluationRequest\x12\x0e\n\x06reason\x18\x01 \x01(\t\"<\n\x18\x43\x61ncelEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t*d\n\x13WeightOperationType\x12\n\n\x06ZEROFY\x10\x00\x12\x10\n\x0cREINITIALIZE\x10\x01\x12\n\n\x06\x46REEZE\x10\x02\x12\x12\n\x0eREMOVE_NEURONS\x10\t\x12\x0f\n\x0b\x41\x44\x44_NEURONS\x10\n*o\n\x0fZerofyPredicate\x12\x19\n\x15ZEROFY_PREDICATE_NONE\x10\x00\x12 \n\x1cZEROFY_PREDICATE_WITH_FROZEN\x10\x01\x12\x1f\n\x1bZEROFY_PREDICATE_WITH_OLDER\x10\x02*M\n\x0f\x41gentIntentType\x12\x12\n\x0eINTENT_UNKNOWN\x10\x00\x12\x11\n\rINTENT_FILTER\x10\x01\x12\x13\n\x0fINTENT_ANALYSIS\x10\x02*I\n\x0eSampleEditType\x12\x11\n\rEDIT_OVERRIDE\x10\x00\x12\x13\n\x0f\x45\x44IT_ACCUMULATE\x10\x01\x12\x0f\n\x0b\x45\x44IT_REMOVE\x10\x02*,\n\x11\x41gentProviderType\x12\x17\n\x13PROVIDER_OPENROUTER\x10\x00\x32\xf5\n\n\x11\x45xperimentService\x12P\n\x13GetLatestLoggerData\x12\x1b.GetLatestLoggerDataRequest\x1a\x1c.GetLatestLoggerDataResponse\x12\x36\n\x11\x45xperimentCommand\x12\x0f.TrainerCommand\x1a\x10.CommandResponse\x12H\n\x11ManipulateWeights\x12\x18.WeightsOperationRequest\x1a\x19.WeightsOperationResponse\x12/\n\nGetWeights\x12\x0f.WeightsRequest\x1a\x10.WeightsResponse\x12\x39\n\x0eGetActivations\x12\x12.ActivationRequest\x1a\x13.ActivationResponse\x12\x37\n\nGetSamples\x12\x13.BatchSampleRequest\x1a\x14.BatchSampleResponse\x12\x37\n\x0e\x41pplyDataQuery\x12\x11.DataQueryRequest\x1a\x12.DataQueryResponse\x12;\n\x0eGetDataSamples\x12\x13.DataSamplesRequest\x1a\x14.DataSamplesResponse\x12\x35\n\x0cGetHistogram\x12\x11.HistogramRequest\x1a\x12.HistogramResponse\x12\x38\n\x0bGetMetaData\x12\x13.GetMetaDataRequest\x1a\x14.GetMetaDataResponse\x12\x37\n\rGetPointCloud\x12\x12.PointCloudRequest\x1a\x10.PointCloudChunk0\x01\x12\x37\n\x0e\x45\x64itDataSample\x12\x11.DataEditsRequest\x1a\x12.DataEditsResponse\x12,\n\rGetDataSplits\x12\x06.Empty\x1a\x13.DataSplitsResponse\x12\x30\n\x10\x43heckAgentHealth\x12\x06.Empty\x1a\x14.AgentHealthResponse\x12\x44\n\x0fInitializeAgent\x12\x17.InitializeAgentRequest\x1a\x18.InitializeAgentResponse\x12G\n\x10\x43hangeAgentModel\x12\x18.ChangeAgentModelRequest\x1a\x19.ChangeAgentModelResponse\x12\x41\n\x0eGetAgentModels\x12\x16.GetAgentModelsRequest\x1a\x17.GetAgentModelsResponse\x12)\n\nResetAgent\x12\x06.Empty\x1a\x13.ResetAgentResponse\x12J\n\x11RestoreCheckpoint\x12\x19.RestoreCheckpointRequest\x1a\x1a.RestoreCheckpointResponse\x12J\n\x11TriggerEvaluation\x12\x19.TriggerEvaluationRequest\x1a\x1a.TriggerEvaluationResponse\x12P\n\x13GetEvaluationStatus\x12\x1b.GetEvaluationStatusRequest\x1a\x1c.GetEvaluationStatusResponse\x12G\n\x10\x43\x61ncelEvaluation\x12\x18.CancelEvaluationRequest\x1a\x19.CancelEvaluationResponseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -37,160 +37,162 @@ _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_options = b'8\001' _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._loaded_options = None _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_options = b'8\001' - _globals['_WEIGHTOPERATIONTYPE']._serialized_start=9734 - _globals['_WEIGHTOPERATIONTYPE']._serialized_end=9834 - _globals['_ZEROFYPREDICATE']._serialized_start=9836 - _globals['_ZEROFYPREDICATE']._serialized_end=9947 - _globals['_AGENTINTENTTYPE']._serialized_start=9949 - _globals['_AGENTINTENTTYPE']._serialized_end=10026 - _globals['_SAMPLEEDITTYPE']._serialized_start=10028 - _globals['_SAMPLEEDITTYPE']._serialized_end=10101 - _globals['_AGENTPROVIDERTYPE']._serialized_start=10103 - _globals['_AGENTPROVIDERTYPE']._serialized_end=10147 + _globals['_WEIGHTOPERATIONTYPE']._serialized_start=9871 + _globals['_WEIGHTOPERATIONTYPE']._serialized_end=9971 + _globals['_ZEROFYPREDICATE']._serialized_start=9973 + _globals['_ZEROFYPREDICATE']._serialized_end=10084 + _globals['_AGENTINTENTTYPE']._serialized_start=10086 + _globals['_AGENTINTENTTYPE']._serialized_end=10163 + _globals['_SAMPLEEDITTYPE']._serialized_start=10165 + _globals['_SAMPLEEDITTYPE']._serialized_end=10238 + _globals['_AGENTPROVIDERTYPE']._serialized_start=10240 + _globals['_AGENTPROVIDERTYPE']._serialized_end=10284 _globals['_GETLATESTLOGGERDATAREQUEST']._serialized_start=46 _globals['_GETLATESTLOGGERDATAREQUEST']._serialized_end=183 _globals['_LOGGERDATAPOINT']._serialized_start=186 _globals['_LOGGERDATAPOINT']._serialized_end=443 _globals['_GETLATESTLOGGERDATARESPONSE']._serialized_start=445 - _globals['_GETLATESTLOGGERDATARESPONSE']._serialized_end=508 - _globals['_EMPTY']._serialized_start=510 - _globals['_EMPTY']._serialized_end=517 - _globals['_NEURONID']._serialized_start=519 - _globals['_NEURONID']._serialized_end=566 - _globals['_WEIGHTOPERATION']._serialized_start=569 - _globals['_WEIGHTOPERATION']._serialized_end=842 - _globals['_WEIGHTSOPERATIONREQUEST']._serialized_start=844 - _globals['_WEIGHTSOPERATIONREQUEST']._serialized_end=939 - _globals['_WEIGHTSOPERATIONRESPONSE']._serialized_start=941 - _globals['_WEIGHTSOPERATIONRESPONSE']._serialized_end=1001 - _globals['_HYPERPARAMETERS']._serialized_start=1004 - _globals['_HYPERPARAMETERS']._serialized_end=1709 - _globals['_METRICSSTATUS']._serialized_start=1711 - _globals['_METRICSSTATUS']._serialized_end=1755 - _globals['_ANNOTATSTATUS']._serialized_start=1757 - _globals['_ANNOTATSTATUS']._serialized_end=1883 - _globals['_ANNOTATSTATUS_METADATAENTRY']._serialized_start=1836 - _globals['_ANNOTATSTATUS_METADATAENTRY']._serialized_end=1883 - _globals['_TRAININGSTATUSEX']._serialized_start=1886 - _globals['_TRAININGSTATUSEX']._serialized_end=2158 - _globals['_HYPERPARAMETERCOMMAND']._serialized_start=2160 - _globals['_HYPERPARAMETERCOMMAND']._serialized_end=2253 - _globals['_DENYSAMPLESOPERATION']._serialized_start=2255 - _globals['_DENYSAMPLESOPERATION']._serialized_end=2317 - _globals['_LOADCHECKPOINTOPERATION']._serialized_start=2319 - _globals['_LOADCHECKPOINTOPERATION']._serialized_end=2367 - _globals['_PLOTNOTEOPERATION']._serialized_start=2369 - _globals['_PLOTNOTEOPERATION']._serialized_end=2467 - _globals['_SAVECHECKPOINTOPERATION']._serialized_start=2469 - _globals['_SAVECHECKPOINTOPERATION']._serialized_end=2545 - _globals['_TRAINERCOMMAND']._serialized_start=2548 - _globals['_TRAINERCOMMAND']._serialized_end=3504 - _globals['_HYPERPARAMETERDESC']._serialized_start=3507 - _globals['_HYPERPARAMETERDESC']._serialized_end=3664 - _globals['_NEURONSTATISTICS']._serialized_start=3667 - _globals['_NEURONSTATISTICS']._serialized_end=4037 - _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_start=3896 - _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_end=3945 - _globals['_LAYERREPRESENTATION']._serialized_start=4040 - _globals['_LAYERREPRESENTATION']._serialized_end=4408 - _globals['_ACTIVATIONREQUEST']._serialized_start=4410 - _globals['_ACTIVATIONREQUEST']._serialized_end=4482 - _globals['_ACTIVATIONMAP']._serialized_start=4484 - _globals['_ACTIVATIONMAP']._serialized_end=4556 - _globals['_ACTIVATIONRESPONSE']._serialized_start=4558 - _globals['_ACTIVATIONRESPONSE']._serialized_end=4658 - _globals['_TASKFIELD']._serialized_start=4661 - _globals['_TASKFIELD']._serialized_end=4808 - _globals['_RECORDMETADATA']._serialized_start=4811 - _globals['_RECORDMETADATA']._serialized_end=5143 - _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_start=5090 - _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_end=5143 - _globals['_SAMPLESTATISTICS']._serialized_start=5146 - _globals['_SAMPLESTATISTICS']._serialized_end=5293 - _globals['_COMMANDRESPONSE']._serialized_start=5296 - _globals['_COMMANDRESPONSE']._serialized_end=5526 - _globals['_SAMPLEREQUEST']._serialized_start=5528 - _globals['_SAMPLEREQUEST']._serialized_end=5613 - _globals['_SAMPLEREQUESTRESPONSE']._serialized_start=5616 - _globals['_SAMPLEREQUESTRESPONSE']._serialized_end=5917 - _globals['_BATCHSAMPLEREQUEST']._serialized_start=5920 - _globals['_BATCHSAMPLEREQUEST']._serialized_end=6066 - _globals['_BATCHSAMPLERESPONSE']._serialized_start=6068 - _globals['_BATCHSAMPLERESPONSE']._serialized_end=6130 - _globals['_WEIGHTSREQUEST']._serialized_start=6132 - _globals['_WEIGHTSREQUEST']._serialized_end=6178 - _globals['_WEIGHTSRESPONSE']._serialized_start=6181 - _globals['_WEIGHTSRESPONSE']._serialized_end=6466 - _globals['_DATAQUERYREQUEST']._serialized_start=6468 - _globals['_DATAQUERYREQUEST']._serialized_end=6550 - _globals['_CATEGORICALTAGDEF']._serialized_start=6552 - _globals['_CATEGORICALTAGDEF']._serialized_end=6605 - _globals['_DATAQUERYRESPONSE']._serialized_start=6608 - _globals['_DATAQUERYRESPONSE']._serialized_end=6905 - _globals['_DATASAMPLESREQUEST']._serialized_start=6908 - _globals['_DATASAMPLESREQUEST']._serialized_end=7102 - _globals['_DATASTAT']._serialized_start=7104 - _globals['_DATASTAT']._serialized_end=7213 - _globals['_DATARECORD']._serialized_start=7215 - _globals['_DATARECORD']._serialized_end=7277 - _globals['_DATASAMPLESRESPONSE']._serialized_start=7279 - _globals['_DATASAMPLESRESPONSE']._serialized_end=7369 - _globals['_HISTOGRAMSUBBAR']._serialized_start=7371 - _globals['_HISTOGRAMSUBBAR']._serialized_end=7438 - _globals['_HISTOGRAMBIN']._serialized_start=7440 - _globals['_HISTOGRAMBIN']._serialized_end=7544 - _globals['_CATEGORICALHISTOGRAMBAR']._serialized_start=7546 - _globals['_CATEGORICALHISTOGRAMBAR']._serialized_end=7637 - _globals['_HISTOGRAMREQUEST']._serialized_start=7639 - _globals['_HISTOGRAMREQUEST']._serialized_end=7691 - _globals['_HISTOGRAMRESPONSE']._serialized_start=7694 - _globals['_HISTOGRAMRESPONSE']._serialized_end=7872 - _globals['_GETMETADATAREQUEST']._serialized_start=7874 - _globals['_GETMETADATAREQUEST']._serialized_end=7961 - _globals['_GETMETADATARESPONSE']._serialized_start=7964 - _globals['_GETMETADATARESPONSE']._serialized_end=8117 - _globals['_POINTCLOUDREQUEST']._serialized_start=8119 - _globals['_POINTCLOUDREQUEST']._serialized_end=8193 - _globals['_POINTCLOUDCHUNK']._serialized_start=8196 - _globals['_POINTCLOUDCHUNK']._serialized_end=8387 - _globals['_DATAEDITSREQUEST']._serialized_start=8390 - _globals['_DATAEDITSREQUEST']._serialized_end=8610 - _globals['_DATAEDITSRESPONSE']._serialized_start=8612 - _globals['_DATAEDITSRESPONSE']._serialized_end=8665 - _globals['_DATASPLITSRESPONSE']._serialized_start=8667 - _globals['_DATASPLITSRESPONSE']._serialized_end=8725 - _globals['_AGENTHEALTHRESPONSE']._serialized_start=8727 - _globals['_AGENTHEALTHRESPONSE']._serialized_end=8784 - _globals['_INITIALIZEAGENTREQUEST']._serialized_start=8786 - _globals['_INITIALIZEAGENTREQUEST']._serialized_end=8880 - _globals['_INITIALIZEAGENTRESPONSE']._serialized_start=8882 - _globals['_INITIALIZEAGENTRESPONSE']._serialized_end=8941 - _globals['_CHANGEAGENTMODELREQUEST']._serialized_start=8943 - _globals['_CHANGEAGENTMODELREQUEST']._serialized_end=8983 - _globals['_CHANGEAGENTMODELRESPONSE']._serialized_start=8985 - _globals['_CHANGEAGENTMODELRESPONSE']._serialized_end=9045 - _globals['_GETAGENTMODELSREQUEST']._serialized_start=9047 - _globals['_GETAGENTMODELSREQUEST']._serialized_end=9070 - _globals['_GETAGENTMODELSRESPONSE']._serialized_start=9072 - _globals['_GETAGENTMODELSRESPONSE']._serialized_end=9146 - _globals['_RESETAGENTRESPONSE']._serialized_start=9148 - _globals['_RESETAGENTRESPONSE']._serialized_end=9202 - _globals['_RESTORECHECKPOINTREQUEST']._serialized_start=9204 - _globals['_RESTORECHECKPOINTREQUEST']._serialized_end=9255 - _globals['_RESTORECHECKPOINTRESPONSE']._serialized_start=9257 - _globals['_RESTORECHECKPOINTRESPONSE']._serialized_end=9318 - _globals['_TRIGGEREVALUATIONREQUEST']._serialized_start=9320 - _globals['_TRIGGEREVALUATIONREQUEST']._serialized_end=9402 - _globals['_TRIGGEREVALUATIONRESPONSE']._serialized_start=9404 - _globals['_TRIGGEREVALUATIONRESPONSE']._serialized_end=9465 - _globals['_GETEVALUATIONSTATUSREQUEST']._serialized_start=9467 - _globals['_GETEVALUATIONSTATUSREQUEST']._serialized_end=9495 - _globals['_GETEVALUATIONSTATUSRESPONSE']._serialized_start=9498 - _globals['_GETEVALUATIONSTATUSRESPONSE']._serialized_end=9627 - _globals['_CANCELEVALUATIONREQUEST']._serialized_start=9629 - _globals['_CANCELEVALUATIONREQUEST']._serialized_end=9670 - _globals['_CANCELEVALUATIONRESPONSE']._serialized_start=9672 - _globals['_CANCELEVALUATIONRESPONSE']._serialized_end=9732 - _globals['_EXPERIMENTSERVICE']._serialized_start=10150 - _globals['_EXPERIMENTSERVICE']._serialized_end=11547 + _globals['_GETLATESTLOGGERDATARESPONSE']._serialized_end=536 + _globals['_EMPTY']._serialized_start=538 + _globals['_EMPTY']._serialized_end=545 + _globals['_NEURONID']._serialized_start=547 + _globals['_NEURONID']._serialized_end=594 + _globals['_WEIGHTOPERATION']._serialized_start=597 + _globals['_WEIGHTOPERATION']._serialized_end=870 + _globals['_WEIGHTSOPERATIONREQUEST']._serialized_start=872 + _globals['_WEIGHTSOPERATIONREQUEST']._serialized_end=967 + _globals['_WEIGHTSOPERATIONRESPONSE']._serialized_start=969 + _globals['_WEIGHTSOPERATIONRESPONSE']._serialized_end=1029 + _globals['_HYPERPARAMETERS']._serialized_start=1032 + _globals['_HYPERPARAMETERS']._serialized_end=1737 + _globals['_METRICSSTATUS']._serialized_start=1739 + _globals['_METRICSSTATUS']._serialized_end=1783 + _globals['_ANNOTATSTATUS']._serialized_start=1785 + _globals['_ANNOTATSTATUS']._serialized_end=1911 + _globals['_ANNOTATSTATUS_METADATAENTRY']._serialized_start=1864 + _globals['_ANNOTATSTATUS_METADATAENTRY']._serialized_end=1911 + _globals['_TRAININGSTATUSEX']._serialized_start=1914 + _globals['_TRAININGSTATUSEX']._serialized_end=2186 + _globals['_HYPERPARAMETERCOMMAND']._serialized_start=2188 + _globals['_HYPERPARAMETERCOMMAND']._serialized_end=2281 + _globals['_DENYSAMPLESOPERATION']._serialized_start=2283 + _globals['_DENYSAMPLESOPERATION']._serialized_end=2345 + _globals['_LOADCHECKPOINTOPERATION']._serialized_start=2347 + _globals['_LOADCHECKPOINTOPERATION']._serialized_end=2395 + _globals['_PLOTNOTEOPERATION']._serialized_start=2397 + _globals['_PLOTNOTEOPERATION']._serialized_end=2495 + _globals['_SAVECHECKPOINTOPERATION']._serialized_start=2497 + _globals['_SAVECHECKPOINTOPERATION']._serialized_end=2573 + _globals['_RESTARTINSTANCEOPERATION']._serialized_start=2575 + _globals['_RESTARTINSTANCEOPERATION']._serialized_end=2601 + _globals['_TRAINERCOMMAND']._serialized_start=2604 + _globals['_TRAINERCOMMAND']._serialized_end=3641 + _globals['_HYPERPARAMETERDESC']._serialized_start=3644 + _globals['_HYPERPARAMETERDESC']._serialized_end=3801 + _globals['_NEURONSTATISTICS']._serialized_start=3804 + _globals['_NEURONSTATISTICS']._serialized_end=4174 + _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_start=4033 + _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_end=4082 + _globals['_LAYERREPRESENTATION']._serialized_start=4177 + _globals['_LAYERREPRESENTATION']._serialized_end=4545 + _globals['_ACTIVATIONREQUEST']._serialized_start=4547 + _globals['_ACTIVATIONREQUEST']._serialized_end=4619 + _globals['_ACTIVATIONMAP']._serialized_start=4621 + _globals['_ACTIVATIONMAP']._serialized_end=4693 + _globals['_ACTIVATIONRESPONSE']._serialized_start=4695 + _globals['_ACTIVATIONRESPONSE']._serialized_end=4795 + _globals['_TASKFIELD']._serialized_start=4798 + _globals['_TASKFIELD']._serialized_end=4945 + _globals['_RECORDMETADATA']._serialized_start=4948 + _globals['_RECORDMETADATA']._serialized_end=5280 + _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_start=5227 + _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_end=5280 + _globals['_SAMPLESTATISTICS']._serialized_start=5283 + _globals['_SAMPLESTATISTICS']._serialized_end=5430 + _globals['_COMMANDRESPONSE']._serialized_start=5433 + _globals['_COMMANDRESPONSE']._serialized_end=5663 + _globals['_SAMPLEREQUEST']._serialized_start=5665 + _globals['_SAMPLEREQUEST']._serialized_end=5750 + _globals['_SAMPLEREQUESTRESPONSE']._serialized_start=5753 + _globals['_SAMPLEREQUESTRESPONSE']._serialized_end=6054 + _globals['_BATCHSAMPLEREQUEST']._serialized_start=6057 + _globals['_BATCHSAMPLEREQUEST']._serialized_end=6203 + _globals['_BATCHSAMPLERESPONSE']._serialized_start=6205 + _globals['_BATCHSAMPLERESPONSE']._serialized_end=6267 + _globals['_WEIGHTSREQUEST']._serialized_start=6269 + _globals['_WEIGHTSREQUEST']._serialized_end=6315 + _globals['_WEIGHTSRESPONSE']._serialized_start=6318 + _globals['_WEIGHTSRESPONSE']._serialized_end=6603 + _globals['_DATAQUERYREQUEST']._serialized_start=6605 + _globals['_DATAQUERYREQUEST']._serialized_end=6687 + _globals['_CATEGORICALTAGDEF']._serialized_start=6689 + _globals['_CATEGORICALTAGDEF']._serialized_end=6742 + _globals['_DATAQUERYRESPONSE']._serialized_start=6745 + _globals['_DATAQUERYRESPONSE']._serialized_end=7042 + _globals['_DATASAMPLESREQUEST']._serialized_start=7045 + _globals['_DATASAMPLESREQUEST']._serialized_end=7239 + _globals['_DATASTAT']._serialized_start=7241 + _globals['_DATASTAT']._serialized_end=7350 + _globals['_DATARECORD']._serialized_start=7352 + _globals['_DATARECORD']._serialized_end=7414 + _globals['_DATASAMPLESRESPONSE']._serialized_start=7416 + _globals['_DATASAMPLESRESPONSE']._serialized_end=7506 + _globals['_HISTOGRAMSUBBAR']._serialized_start=7508 + _globals['_HISTOGRAMSUBBAR']._serialized_end=7575 + _globals['_HISTOGRAMBIN']._serialized_start=7577 + _globals['_HISTOGRAMBIN']._serialized_end=7681 + _globals['_CATEGORICALHISTOGRAMBAR']._serialized_start=7683 + _globals['_CATEGORICALHISTOGRAMBAR']._serialized_end=7774 + _globals['_HISTOGRAMREQUEST']._serialized_start=7776 + _globals['_HISTOGRAMREQUEST']._serialized_end=7828 + _globals['_HISTOGRAMRESPONSE']._serialized_start=7831 + _globals['_HISTOGRAMRESPONSE']._serialized_end=8009 + _globals['_GETMETADATAREQUEST']._serialized_start=8011 + _globals['_GETMETADATAREQUEST']._serialized_end=8098 + _globals['_GETMETADATARESPONSE']._serialized_start=8101 + _globals['_GETMETADATARESPONSE']._serialized_end=8254 + _globals['_POINTCLOUDREQUEST']._serialized_start=8256 + _globals['_POINTCLOUDREQUEST']._serialized_end=8330 + _globals['_POINTCLOUDCHUNK']._serialized_start=8333 + _globals['_POINTCLOUDCHUNK']._serialized_end=8524 + _globals['_DATAEDITSREQUEST']._serialized_start=8527 + _globals['_DATAEDITSREQUEST']._serialized_end=8747 + _globals['_DATAEDITSRESPONSE']._serialized_start=8749 + _globals['_DATAEDITSRESPONSE']._serialized_end=8802 + _globals['_DATASPLITSRESPONSE']._serialized_start=8804 + _globals['_DATASPLITSRESPONSE']._serialized_end=8862 + _globals['_AGENTHEALTHRESPONSE']._serialized_start=8864 + _globals['_AGENTHEALTHRESPONSE']._serialized_end=8921 + _globals['_INITIALIZEAGENTREQUEST']._serialized_start=8923 + _globals['_INITIALIZEAGENTREQUEST']._serialized_end=9017 + _globals['_INITIALIZEAGENTRESPONSE']._serialized_start=9019 + _globals['_INITIALIZEAGENTRESPONSE']._serialized_end=9078 + _globals['_CHANGEAGENTMODELREQUEST']._serialized_start=9080 + _globals['_CHANGEAGENTMODELREQUEST']._serialized_end=9120 + _globals['_CHANGEAGENTMODELRESPONSE']._serialized_start=9122 + _globals['_CHANGEAGENTMODELRESPONSE']._serialized_end=9182 + _globals['_GETAGENTMODELSREQUEST']._serialized_start=9184 + _globals['_GETAGENTMODELSREQUEST']._serialized_end=9207 + _globals['_GETAGENTMODELSRESPONSE']._serialized_start=9209 + _globals['_GETAGENTMODELSRESPONSE']._serialized_end=9283 + _globals['_RESETAGENTRESPONSE']._serialized_start=9285 + _globals['_RESETAGENTRESPONSE']._serialized_end=9339 + _globals['_RESTORECHECKPOINTREQUEST']._serialized_start=9341 + _globals['_RESTORECHECKPOINTREQUEST']._serialized_end=9392 + _globals['_RESTORECHECKPOINTRESPONSE']._serialized_start=9394 + _globals['_RESTORECHECKPOINTRESPONSE']._serialized_end=9455 + _globals['_TRIGGEREVALUATIONREQUEST']._serialized_start=9457 + _globals['_TRIGGEREVALUATIONREQUEST']._serialized_end=9539 + _globals['_TRIGGEREVALUATIONRESPONSE']._serialized_start=9541 + _globals['_TRIGGEREVALUATIONRESPONSE']._serialized_end=9602 + _globals['_GETEVALUATIONSTATUSREQUEST']._serialized_start=9604 + _globals['_GETEVALUATIONSTATUSREQUEST']._serialized_end=9632 + _globals['_GETEVALUATIONSTATUSRESPONSE']._serialized_start=9635 + _globals['_GETEVALUATIONSTATUSRESPONSE']._serialized_end=9764 + _globals['_CANCELEVALUATIONREQUEST']._serialized_start=9766 + _globals['_CANCELEVALUATIONREQUEST']._serialized_end=9807 + _globals['_CANCELEVALUATIONRESPONSE']._serialized_start=9809 + _globals['_CANCELEVALUATIONRESPONSE']._serialized_end=9869 + _globals['_EXPERIMENTSERVICE']._serialized_start=10287 + _globals['_EXPERIMENTSERVICE']._serialized_end=11684 # @@protoc_insertion_point(module_scope) diff --git a/weightslab/proto/experiment_service_pb2_grpc.py b/weightslab/proto/experiment_service_pb2_grpc.py index e75d391c..440a5a51 100644 --- a/weightslab/proto/experiment_service_pb2_grpc.py +++ b/weightslab/proto/experiment_service_pb2_grpc.py @@ -5,7 +5,7 @@ from weightslab.proto import experiment_service_pb2 as weightslab_dot_proto_dot_experiment__service__pb2 -GRPC_GENERATED_VERSION = '1.76.0' +GRPC_GENERATED_VERSION = '1.68.1' GRPC_VERSION = grpc.__version__ _version_not_supported = False @@ -18,7 +18,7 @@ if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' - + ' but the generated code in weightslab/proto/experiment_service_pb2_grpc.py depends on' + + f' but the generated code in weightslab/proto/experiment_service_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' diff --git a/weightslab/security/cert_auth_manager.py b/weightslab/security/cert_auth_manager.py index 85c63b34..168072c4 100644 --- a/weightslab/security/cert_auth_manager.py +++ b/weightslab/security/cert_auth_manager.py @@ -123,24 +123,34 @@ def generate_certs(self, force: bool = False) -> Tuple[bool, str]: logger.info(f"Certificates already exist in {self.certs_dir}") return True, f"Certificates already exist in {self.certs_dir}" - # Try to use PowerShell to generate certs + # Use the bundled cert-generation scripts under weightslab/ui/utils. try: - script_dir = Path(__file__).parent.parent.parent / 'docker' / 'docker' / 'utils' - generate_script = script_dir / 'generate-certs.ps1' - - if not generate_script.exists(): - return False, f"Certificate generation script not found at {generate_script}" - - cmd = ['powershell', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', str(generate_script)] - if force: - cmd.append('-force_create_certs') + script_dir = Path(__file__).parent.parent / 'ui' / 'utils' + env = os.environ.copy() + env['WEIGHTSLAB_CERTS_DIR'] = str(self.certs_dir) + + if os.name == 'nt': + generate_script = script_dir / 'generate-certs-auth-token.ps1' + if not generate_script.exists(): + return False, f"Certificate generation script not found at {generate_script}" + cmd = ['powershell', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', str(generate_script)] + if force: + cmd.append('-ForceCreateCerts') + else: + generate_script = script_dir / 'generate-certs-auth-token.sh' + if not generate_script.exists(): + return False, f"Certificate generation script not found at {generate_script}" + cmd = ['bash', str(generate_script)] + if force: + cmd.append('--force-create-certs') result = subprocess.run( cmd, capture_output=True, text=True, check=False, - cwd=str(script_dir) + cwd=str(script_dir), + env=env, ) if result.returncode != 0: @@ -214,10 +224,6 @@ def setup_tls_environment(self) -> dict: 'GRPC_TLS_KEY_FILE': str(self.key_file), 'GRPC_TLS_CA_FILE': str(self.ca_file), 'WEIGHTSLAB_CERTS_DIR': str(self.certs_dir), - 'ENVOY_DOWNSTREAM_TLS': 'on', - 'ENVOY_UPSTREAM_TLS': 'on', - 'WS_SERVER_PROTOCOL': 'https', - 'VITE_SERVER_PROTOCOL': 'https', } return env_vars @@ -234,12 +240,9 @@ def setup_auth_environment(self) -> dict: if self.enable_auth: token = self.get_or_create_auth_token() env_vars['WL_ENABLE_GRPC_AUTH_TOKEN'] = '1' - env_vars['VITE_WL_ENABLE_GRPC_AUTH_TOKEN'] = '1' env_vars['GRPC_AUTH_TOKEN'] = token - env_vars['VITE_GRPC_AUTH_TOKEN'] = token else: env_vars['WL_ENABLE_GRPC_AUTH_TOKEN'] = '0' - env_vars['VITE_WL_ENABLE_GRPC_AUTH_TOKEN'] = '0' return env_vars @@ -252,7 +255,7 @@ def check_and_apply(self) -> Tuple[bool, str]: Tuple of (success, message) """ if not self.has_valid_certs(): - return False, f"No certs found in {self.certs_dir}. Run: weightslab ui se" + return False, f"No certs found in {self.certs_dir}. Run: weightslab se" env_vars = self.setup_tls_environment() env_vars.update(self.setup_auth_environment()) @@ -266,7 +269,7 @@ def check_and_apply(self) -> Tuple[bool, str]: def setup_secure_environment(self, force_certs: bool = False) -> Tuple[bool, str]: """ Generate TLS certificates and auth token. - Called explicitly by 'weightslab ui se' command. + Called explicitly by 'weightslab se' command. Args: force_certs: Force regenerate certificates diff --git a/weightslab/src.py b/weightslab/src.py index aa5a39b0..5601c867 100644 --- a/weightslab/src.py +++ b/weightslab/src.py @@ -14,6 +14,7 @@ import tempfile import functools import threading +import queue as _queue import traceback import numpy as np import torch as th @@ -24,12 +25,12 @@ from weightslab.backend.dataloader_interface import DataLoaderInterface from weightslab.components.checkpoint_manager import CheckpointManager from weightslab.backend.optimizer_interface import OptimizerInterface -from weightslab.backend.ledgers import DEFAULT_NAME, get_checkpoint_manager, get_model, get_dataloader, get_dataframe, get_optimizer, register_hyperparams, watch_hyperparams_file, get_hyperparams, register_logger, get_logger, register_signal, get_signal, list_models +from weightslab.backend.ledgers import DEFAULT_NAME, get_checkpoint_manager, get_model, get_dataloader, get_dataframe, get_optimizer, register_hyperparams, watch_hyperparams_file, get_hyperparams, register_logger, get_logger, register_signal, get_signal, list_models, has_serving_config from weightslab.backend.model_interface import ModelInterface from weightslab.trainer.trainer_services import grpc_serve from weightslab.data.sample_stats import SampleStatsEx from weightslab.utils.logs import set_log_directory -from weightslab.utils.tools import detach_to_cpu +from weightslab.utils.tools import detach_to_cpu, _running_in_notebook from weightslab.backend.logger import LoggerQueue from weightslab.backend.cli import cli_serve from weightslab.backend import ledgers @@ -71,6 +72,231 @@ def _rebind_caller_local(original_obj: Any, new_obj: Any) -> None: DATAFRAME_M = None # Global registry for custom signals _REGISTERED_SIGNALS = {} +_REACTIVE_FIRED = {} # signal_name -> step it last fired at (per-step dedup) + + +def _gather_inputs_fresh(lg, inputs, ids, step, fresh_cache=None): + """``{name: (B,) array}`` if every input has a value at *step* for every id, + else ``None`` (value AT step, not latest — a lagging worker still gathers it). + Reads via ``query_per_sample_at_step`` (O(batch), cached). + + *fresh_cache*: in-memory values fired this pass — read them so the pass + persists once at the end, not per signal.""" + cols = {} + for inp in inputs: + if fresh_cache is not None and inp in fresh_cache: + cols[inp] = fresh_cache[inp] + continue + # A reactive-derived input lives only in fresh_cache this pass (it's not + # in the ledger at this step until the end-of-pass persist). If it's not + # there yet, skip — don't query the ledger (a guaranteed-miss flush). + _m = getattr(_REGISTERED_SIGNALS.get(inp), '_wl_signal_meta', {}) + if _m.get('inputs'): + return None + at = {int(sid): val for sid, val in lg.query_per_sample_at_step(inp, ids, step)} + for sid in ids: + if sid not in at: + return None + cols[inp] = np.array([at[sid] for sid in ids], dtype=float) + return cols + + +def _react_dependents(seed_names, batch_ids, step, origin='train'): + """Reactive signal firing. When the signals in *seed_names* were just logged, + fire any ``@wl.signal(inputs=[...])`` whose inputs are now ALL present at + *step* — order-independently, regardless of which input arrived last. Each + fired signal is itself a seed (chaining) and fires at most once per step. + Inputs are read from the logger, so ingested signals must be logged.""" + _warn_on_signal_cycles() # lazy, once/process — fires even with no wl.serve + if batch_ids is None: + return + try: + ids = [int(u) for u in (batch_ids.detach().cpu().numpy() + if hasattr(batch_ids, 'detach') else batch_ids)] + except Exception: + return + lg = get_logger() + if lg is None: + return + try: + df_proxy = get_dataframe() + except Exception: + df_proxy = None + + def dependents_of(nm): + for name, func in list(_REGISTERED_SIGNALS.items()): + meta = getattr(func, '_wl_signal_meta', {}) + inputs = meta.get('inputs') + if inputs and nm in inputs: + yield name, func, meta, inputs + + # Fired this pass; kept in-memory so chains read without a ledger round-trip + # and the whole pass persists in ONE save_signals (not one write per signal). + fired = {} # {name: (B,) array} + fired_log = {} # {name: log flag} + queue = list(seed_names) + while queue: + logged = queue.pop() + for name, func, meta, inputs in dependents_of(logged): + if _REACTIVE_FIRED.get(name) == step: + continue + every = meta.get('compute_every_n_steps', 1) or 1 + if step % every != 0: + continue + cols = _gather_inputs_fresh(lg, inputs, ids, step, fresh_cache=fired) + if cols is None: + continue # inputs not all fresh yet — a later log will re-trigger + _REACTIVE_FIRED[name] = step + try: + if meta.get('batched'): + bctx = BatchSignalContext(sample_ids=ids, subscribed_values=cols[inputs[0]], + logger=lg, dataframe=df_proxy, origin=origin, step=step, + inputs=cols) + res = func(bctx) + vals = res.tolist() if hasattr(res, 'tolist') else list(res) + else: + vals = [] + for i, sid in enumerate(ids): + ctx = SignalContext(sample_id=sid, subscribed_value=float(cols[inputs[0]][i]), + logger=lg, dataframe=df_proxy, origin=origin, step=step, + inputs={k: float(cols[k][i]) for k in cols}) + vals.append(func(ctx)) + vals = [float(x) for x in vals] + except StaleSignalError: + raise + except Exception as e: + logger.debug(f"reactive signal '{name}' failed: {e}") + continue + fired[name] = np.asarray(vals, dtype=float) + fired_log[name] = bool(meta.get('log', True)) + queue.append(name) + + # One persist for all signals fired this pass. step=step attributes values to + # the firing step (matters when the worker lags). _react=False: no re-dispatch. + if fired: + try: + _logged = {n: v.tolist() for n, v in fired.items() if fired_log.get(n, True)} + _unlogged = {n: v.tolist() for n, v in fired.items() if not fired_log.get(n, True)} + if _logged: + save_signals(signals=_logged, batch_ids=ids, step=step, log=True, _react=False) + if _unlogged: + save_signals(signals=_unlogged, batch_ids=ids, step=step, log=False, _react=False) + except Exception as e: + logger.debug(f"reactive batched persist failed: {e}") + + +def _detect_signal_cycles(): + """Cycles in the reactive ``inputs`` graph, each as a name list ending where + it began. Only edges to other registered signals count (base metrics are + leaves). A signal in a cycle never fires — surfacing it beats silent no-fire.""" + graph = {} + for name, func in _REGISTERED_SIGNALS.items(): + meta = getattr(func, '_wl_signal_meta', {}) + inputs = meta.get('inputs') or [] + graph[name] = [dep for dep in inputs if dep in _REGISTERED_SIGNALS] # depends-on edges + WHITE, GREY, BLACK = 0, 1, 2 + color = {n: WHITE for n in graph} + stack, cycles, seen = [], [], set() + + def dfs(n): + color[n] = GREY + stack.append(n) + for dep in graph.get(n, ()): + if color.get(dep) == GREY: # back-edge → cycle + cyc = stack[stack.index(dep):] + [dep] + key = frozenset(cyc) + if key not in seen: + seen.add(key) + cycles.append(cyc) + elif color.get(dep) == WHITE: + dfs(dep) + stack.pop() + color[n] = BLACK + + for n in graph: + if color[n] == WHITE: + dfs(n) + return cycles + + +_SIGNAL_CYCLE_CHECKED = False + + +def _warn_on_signal_cycles(force=False): + """Warn once per process for each reactive cycle. Fired from start_training + and the first dispatch (so it works headless). Warns, doesn't raise.""" + global _SIGNAL_CYCLE_CHECKED + if _SIGNAL_CYCLE_CHECKED and not force: + return + _SIGNAL_CYCLE_CHECKED = True + for cyc in _detect_signal_cycles(): + logger.warning( + "WeightsLab reactive signals: circular dependency %s — these signals " + "will NEVER fire (each waits on the other's value, which never becomes " + "fresh). Break the cycle in their @wl.signal(inputs=[...]).", + " -> ".join(cyc)) + + +# --- optional signal-worker thread: run reactive dispatch off the train thread - +_SIGNAL_WORKER = {"q": None, "thread": None} + + +def _signal_worker_enabled(): + try: + return bool(get_hyperparams().get("ledger_signal_worker", False)) + except Exception: + return False + + +def _ensure_signal_worker(): + if _SIGNAL_WORKER["thread"] is not None and _SIGNAL_WORKER["thread"].is_alive(): + return _SIGNAL_WORKER["q"] + q = _queue.Queue() + + def _worker(): + while True: + job = q.get() + if job is None: + q.task_done() + break + seed_names, ids, step, origin = job + try: + _react_dependents(seed_names, ids, step, origin) + except Exception as e: + logger.debug(f"signal worker job failed: {e}") + finally: + q.task_done() + + t = threading.Thread(target=_worker, name="WL-Signal-Worker", daemon=True) + t.start() + _SIGNAL_WORKER["q"] = q + _SIGNAL_WORKER["thread"] = t + return q + + +def _dispatch_or_enqueue(seed_names, batch_ids, step, origin): + """Reactive dispatch: inline on the train thread, or handed to the signal + worker thread when ``ledger_signal_worker`` is set. The seed signals are + already logged synchronously, so the worker only defers the derived compute + + persistence — the inputs it reads are already in the ledger.""" + if not _signal_worker_enabled(): + _react_dependents(seed_names, batch_ids, step, origin) + return + try: + ids = [int(u) for u in (batch_ids.detach().cpu().numpy() + if hasattr(batch_ids, 'detach') else batch_ids)] + except Exception: + return + _ensure_signal_worker().put((list(seed_names), ids, step, origin)) + + +def drain_signals(): + """Block until the signal-worker thread has processed all queued reactive + jobs. Called automatically by :func:`write_dataframe`; call it manually + before reading signals mid-run when ``ledger_signal_worker`` is enabled.""" + q = _SIGNAL_WORKER["q"] + if q is not None: + q.join() # Evaluation function registered via the @wl.eval_fn decorator. _REGISTERED_EVAL_FN: Optional[Any] = None @@ -78,18 +304,36 @@ def _rebind_caller_local(original_obj: Any, new_obj: Any) -> None: _EVAL_WORKER_THREAD: Optional[threading.Thread] = None +class StaleSignalError(RuntimeError): + """Raised when a signal ingests another signal that is not *fresh* — i.e. the + ingested signal has no value at the current step (it was not logged, or was + written after the trigger this step). See ``SignalContext.latest`` / + ``BatchSignalContext.latest`` (``require_fresh=True``) and + ``@wl.signal(ingests=[...])``.""" + pass + + class SignalContext: """ Unified context object for WeightsLab signals. Carries all available metadata for a single sample during computation. """ - def __init__(self, sample_id, dataframe, data=None, subscribed_value=None, logger=None, origin=None): + def __init__(self, sample_id, dataframe, data=None, subscribed_value=None, logger=None, origin=None, + step=None, logits=None, preds=None, targets=None, inputs=None): self.sample_id = sample_id self.dataframe = dataframe self.data = data self.subscribed_value = subscribed_value self.logger = logger self.origin = origin + self.step = step # step the trigger fired at (for freshness checks) + self.logits = logits # this sample's raw model output row (subscribe_to path) + self.preds = preds + self.targets = targets + # Current-step value of each declared @wl.signal(inputs=[...]) input for + # THIS sample, keyed by signal name: ``ctx.inputs["sig/entropy"]``. Empty + # for signals that don't declare inputs (e.g. the subscribe_to path). + self.inputs = inputs if inputs is not None else {} @property def image(self) -> Optional[np.ndarray]: @@ -163,6 +407,91 @@ def is_dynamic(self) -> bool: """True if running during training (triggered by a metric).""" return self.subscribed_value is not None + def latest(self, signal_name, default=float("nan"), require_fresh=False): + """Most recent value of ANOTHER signal for THIS sample. Lets a signal + ingest several other signals: read each with ``ctx.latest(name)`` and + combine. Returns *default* if the signal has no value yet. + + With ``require_fresh=True`` this raises if the ingested signal has no + value at the current step (``self.step``) — i.e. it was not logged, or + was written after (not before) the trigger this step. + """ + if self.logger is None: + return default + rows = self.logger.query_per_sample(signal_name, sample_ids=[self.sample_id]) + if require_fresh and (not rows or self.step is None or rows[-1][1] != self.step): + raise StaleSignalError( + f"ingested signal '{signal_name}' is not fresh at step {self.step} for " + f"sample {self.sample_id} (log it with log=True and write it BEFORE the " + f"subscribing signal fires).") + return rows[-1][2] if rows else default # rows ordered by seq -> last is newest + + +class BatchSignalContext: + """Batched context for a vectorized ``@wl.signal(batched=True)``. + + Instead of one :class:`SignalContext` per sample, a batched signal receives + the whole batch at once: ``sample_ids`` and ``subscribed_values`` are arrays + of length B, so the signal computes over all samples with vector ops and + returns one array of length B. It also exposes **batched** ledger reads + (:meth:`history` runs a single query for the whole batch instead of one query + per sample), which is where the large speed-ups come from. + """ + def __init__(self, sample_ids, subscribed_values, logger=None, dataframe=None, origin=None, + step=None, logits=None, preds=None, targets=None, inputs=None): + self.sample_ids = [int(s) for s in sample_ids] # length B + self.subscribed_values = np.asarray(subscribed_values, dtype=float) # (B,) + self.logger = logger + self.dataframe = dataframe + self.origin = origin + self.step = step # step the trigger fired at (for freshness checks) + # Raw model output for this batch (subscribe_to path): logit-derived + # signals compute from these, no save_signals. None on the inputs=[...] path. + self.logits = logits # (B, C) + self.preds = preds + self.targets = targets + # Current-step values of each declared @wl.signal(inputs=[...]) input for + # the whole batch, keyed by signal name: ``ctx.inputs["sig/entropy"]`` is a + # ``(B,)`` array aligned to ``sample_ids``. Populated by the reactive + # dispatch; empty for signals that don't declare inputs. + self.inputs = inputs if inputs is not None else {} + + def history(self, signal_name): + """Per-sample history of *signal_name* for every sample in the batch, in + a SINGLE ledger query. Returns ``{sample_id: [values in step order]}`` + (empty list for samples with no history yet).""" + out = {s: [] for s in self.sample_ids} + if self.logger is None: + return out + # query_per_sample accepts a list of ids -> one scan for the whole batch. + for sid, step, val, _ in self.logger.query_per_sample(signal_name, sample_ids=self.sample_ids): + out.setdefault(int(sid), []).append(val) # rows already ordered by seq (= step order) + return out + + def latest(self, signal_name, default=float("nan"), require_fresh=False): + """Most recent value of ANOTHER signal for each sample in the batch, in a + single batched query. ``(B,)`` array aligned to ``self.sample_ids``. Use + this to build a signal that ingests several other signals — call it once + per input signal, then combine the arrays with vector ops. + + With ``require_fresh=True`` this raises unless EVERY sample has a value of + *signal_name* at the current step (``self.step``) — catching a stale + ingest (input not logged, or written after the trigger this step). + """ + last = {} + if self.logger is not None: + for sid, step, val, _ in self.logger.query_per_sample(signal_name, sample_ids=self.sample_ids): + last[int(sid)] = (step, val) # rows ordered by seq -> last assignment wins + if require_fresh: + stale = [s for s in self.sample_ids + if s not in last or self.step is None or last[s][0] != self.step] + if stale: + raise StaleSignalError( + f"ingested signal '{signal_name}' is not fresh at step {self.step} for " + f"{len(stale)}/{len(self.sample_ids)} sample(s) (e.g. {stale[:3]}). Log it " + f"with log=True and write it BEFORE the subscribing signal fires.") + return np.array([last.get(s, (None, default))[1] for s in self.sample_ids], dtype=float) + # ##################################################################################################################### # WEIGHTSLAB INTERNAL FUNCTIONS FOR LOGGING, SIGNAL EXTRACTION, WRAPPING, ETC. (not typically called directly by users) @@ -273,9 +602,21 @@ def _extract_scalar_from_tensor(batch_scalar: th.Tensor | np.ndarray, out: th.Te batch_scalar = _tmp except Exception: pass - # Merged batch scalar with ids + # Merged batch scalar with ids. Vectorize the tensor->python + # conversion: one .tolist() (a single device sync) instead of a + # per-element .item() in a comprehension (B device syncs + B python + # calls). This is on the per-step hot path — profiling showed the + # per-element version was a top self-time cost. if isinstance(batch_scalar, (th.Tensor, np.ndarray)) and ids is not None and len(batch_scalar) == len(ids): - batch_scalar = {ids[i].item() if isinstance(ids, th.Tensor) else ids[i]: batch_scalar[i].item() for i in range(len(batch_scalar))} + _vals = (batch_scalar.detach().cpu().tolist() if isinstance(batch_scalar, th.Tensor) + else np.asarray(batch_scalar).tolist()) + if isinstance(ids, th.Tensor): + _keys = ids.detach().cpu().tolist() + elif isinstance(ids, np.ndarray): + _keys = ids.tolist() + else: + _keys = list(ids) + batch_scalar = dict(zip(_keys, _vals)) # 2. Otherwise fall back to extracting from 'out' elif out is not None: if isinstance(out, th.Tensor): @@ -466,6 +807,8 @@ def wrappered_fwd(original_forward, kwargs, reg_name, *a, **kw): # Extract torch function parameters _ = wl_kw.get('flag') + # Kept in-memory for ctx.logits (logit-derived signals); NOT persisted unless + # ledger_store_preds_raw is set (see the gated save_signals below). preds_raw = a[0] if len(a) > 0 else None # User parameters @@ -620,39 +963,87 @@ def wrappered_fwd(original_forward, kwargs, reg_name, *a, **kw): for name, func in subscribers: meta = getattr(func, '_wl_signal_meta', {}) compute_every = meta.get('compute_every_n_steps', 1) + min_step = meta.get('min_step', 0) + + # Warm-up gate: don't fire until we've reached min_step + # (e.g. a signal that needs enough per-sample history). + if current_step < min_step: + continue # Frequency Check if current_step % compute_every != 0: continue try: - batch_res = {} - for i, uid in enumerate(ids_np): - # Generic 'value' argument - val = float(val_vec[i]) - - # Unified Context Pattern - ctx = SignalContext( - sample_id=int(uid), - subscribed_value=val, - logger=ledgers.get_logger(), + _lg = ledgers.get_logger() + ingests = meta.get('ingests') or [] + if meta.get('batched'): + # Vectorized path: build ONE context for the whole + # batch and call the signal once. It returns a length-B + # array. Avoids B Python calls + B SignalContext allocs, + # and lets the signal do batched ledger reads. + bctx = BatchSignalContext( + sample_ids=[int(u) for u in ids_np], + subscribed_values=[float(v) for v in val_vec], + logger=_lg, dataframe=df_proxy, - origin=kwargs.get('origin', 'train') + origin=kwargs.get('origin', 'train'), + step=step, + logits=preds_raw.detach() if hasattr(preds_raw, 'detach') else preds_raw, + preds=preds.detach() if hasattr(preds, 'detach') else preds, + targets=targets.detach() if hasattr(targets, 'detach') else targets, ) - try: - res = func(ctx) # Compute per sample result with unified context - except TypeError: - # Fallback for legacy subscriber functions - res = func(sample_id=int(uid), value=val, dataframe=df_proxy) - - batch_res[uid] = res - signal_value = list(batch_res.values()) + # Enforce that declared ingested signals are fresh. + for dep in ingests: + bctx.latest(dep, require_fresh=True) + res = func(bctx) + res = res.tolist() if hasattr(res, 'tolist') else list(res) + signal_value = [float(x) for x in res] + else: + # Validate declared ingests once for the whole batch. + if ingests: + _vctx = BatchSignalContext( + sample_ids=[int(u) for u in ids_np], + subscribed_values=[float(v) for v in val_vec], + logger=_lg, step=step) + for dep in ingests: + _vctx.latest(dep, require_fresh=True) + batch_res = {} + for i, uid in enumerate(ids_np): + # Generic 'value' argument + val = float(val_vec[i]) + + # Unified Context Pattern + _lrow = preds_raw[i] if hasattr(preds_raw, '__getitem__') and preds_raw is not None else None + ctx = SignalContext( + sample_id=int(uid), + subscribed_value=val, + logger=_lg, + dataframe=df_proxy, + origin=kwargs.get('origin', 'train'), + step=step, + logits=_lrow.detach() if hasattr(_lrow, 'detach') else _lrow, + ) + try: + res = func(ctx) # Compute per sample result with unified context + except TypeError: + # Fallback for legacy subscriber functions + res = func(sample_id=int(uid), value=val, dataframe=df_proxy) + + batch_res[uid] = res + signal_value = list(batch_res.values()) dynamic_updates[name] = signal_value if dynamic_updates and meta.get('log', True): logger.debug(f"Dynamic updates computed for signal '{reg_name}': {list(dynamic_updates.keys())}") - _log_signal(sum(signal_value)/len(signal_value), signal_value, name, step=step, **kwargs) # Log custom subscribed signals + # Log per-sample keyed by id (dict), not a bare list — + # else it reaches the dataframe but not the logger's + # per_sample table, and reactive dependents can't read it. + _sv = {int(ids_np[i]): float(signal_value[i]) for i in range(len(ids_np))} + _log_signal(sum(signal_value)/len(signal_value), _sv, name, step=step, **kwargs) # Log custom subscribed signals + except StaleSignalError: + raise # correctness enforcement: surface, don't swallow except Exception as e: - logger.debug(f"Dynamic signal {name} failed: {e}") + logger.error(f"Dynamic signal {name} failed: {e}") pass # User function error, skip # Save statistics if requested and applicable. @@ -667,18 +1058,37 @@ def wrappered_fwd(original_forward, kwargs, reg_name, *a, **kw): signals.update(dynamic_updates) # Merge dynamic signals preds = detach_to_cpu(preds) - preds_raw = detach_to_cpu(preds_raw) + # preds_raw = detach_to_cpu(preds_raw) + + # Storing preds_raw (the (B,C) logits) every step is costly and only + # needed for FiftyOne/report drill-down — signals already read them via + # ctx.logits. Default off (dev disabled it); set the hyperparam to re-enable. + try: + _store_preds = bool(get_hyperparams().get('ledger_store_preds_raw', False)) + except Exception: + _store_preds = False # Enqueue signals and data save_signals( signals=signals, batch_ids=batch_ids, - preds_raw=preds_raw, + preds_raw=preds_raw if _store_preds else None, preds=preds, targets=targets, log=False # Already logged above, no need to log again in save_signals; set to False to avoid duplicate logging if save_signals is called separately without logging ) + # Seed reactive dispatch with the watched metric AND subscribe_to results, so + # a chain like conf_r<-entropy<-logits fires without any user save_signals. + if not per_instance and batch_ids is not None: + try: + _seed = [reg_name] + [n for n in dynamic_updates.keys() if n != reg_name] + _dispatch_or_enqueue(_seed, batch_ids, step, origin or 'train') + except StaleSignalError: + raise + except Exception as e: + logger.debug(f"reactive dispatch after {reg_name} failed: {e}") + # Return the original output (dict for per-instance losses so caller can # use out['batch'] for backward, tensor for standard per-sample losses). return out_original if isinstance(out_original, dict) else out @@ -869,6 +1279,14 @@ def new_forward(*a, **kw): except Exception: pass + # Auto-detect for background loss-shape classification: any signal + # registered specifically via flag="loss" is, by convention, a + # reduction="none" per-sample criterion — exactly what the loss-shape + # classifier expects. No opt-in call needed; see + # auto_loss_shape_signal_names / LoggerQueue._autotag_loss_shapes. + if reg_name and 'loss' in flag.lower(): + _AUTO_LOSS_SHAPE_SIGNALS.add(reg_name) + # rebind caller, i.e., in-place update _loss = get_signal(reg_name) _rebind_caller_local(obj, _loss) @@ -1068,38 +1486,132 @@ def start_training(timeout: int = None) -> None: timeout: Maximum number of seconds to keep running. If ``None``, runs until interrupted. """ + _warn_on_signal_cycles() # surface circular reactive-signal deps (warn, don't raise) if timeout is not None and isinstance(timeout, int) and timeout > 0: logger.info(f"Starting WeightsLab training mode with a timeout of {timeout} seconds.") time.sleep(timeout) pause_ctrl.resume() # Ensure we're not paused if start_training is called after serve + def serve(serving_cli: bool = True, serving_grpc: bool = False, - spawn_cli_client: bool = False, **kwargs) -> None: + spawn_cli_client: bool = False, serving_bore: bool = False, + bore_port: int = None, allow_unconfigured: bool = True, **kwargs): """Start WeightsLab services. Args: serving_cli: Start the interactive CLI server. - serving_grpc: Start the gRPC server. + serving_grpc: Start the gRPC server. In a notebook/Colab, if this is on + but ``serving_bore`` is off, a warning suggests ``serving_bore=True`` + (the UI runs on your own machine and needs a tunnel to reach here). spawn_cli_client: When ``serving_cli`` is True, also open the interactive REPL in a new console window (the default, backward-compatible behavior). Set to ``False`` to start the CLI server *headless* — it still advertises its port, so you can attach on demand from any terminal with ``weightslab cli`` (or ``weightslab cli --port ``). + serving_bore: Expose the gRPC backend to the internet via a zero-signup + `bore `_ raw-TCP relay, so a Weights + Studio running on another machine can reach it (e.g. training in + Google Colab, UI on your laptop). Prints the public endpoint and the + two commands to run locally (``weightslab start`` + + ``weightslab tunnel ``). Implies a gRPC backend. Uses the + shared public relay ``bore.pub`` — the random port is the only thing + guarding it, so keep it to non-sensitive demos. + bore_port: Port to expose when ``serving_bore`` is set. Defaults to the + gRPC port (``grpc_port`` kwarg, else ``$GRPC_BACKEND_PORT``, else + 50051). + allow_unconfigured: Escape hatch for the serving-config guard. ``serve`` + reads hyperparameters once, here, for TLS, ports and + ``root_log_dir``; if none were wrapped + (``wl.watch_or_edit(params, flag="hyperparameters")``) those settings + silently fall back to env vars / defaults. Serving over the network + (``serving_grpc`` / ``serving_bore``) without config therefore + raises, because the backend may come up without TLS/auth. Pass + ``True`` (or set ``WL_ALLOW_UNCONFIGURED_SERVE=1``) to downgrade the + error to a warning and serve anyway. CLI-only serving always warns + (never raises), regardless of this flag. **kwargs: Extra server options passed to underlying backends (e.g. ``cli_host``, ``cli_port``, ``grpc_port``). + + Returns: + The public ``host:port`` bore endpoint string when ``serving_bore`` is + set and the tunnel came up, otherwise ``None``. + + Raises: + RuntimeError: If ``serving_grpc`` or ``serving_bore`` is requested before + any serving config was wrapped and the escape hatch is not set. """ + # ------------------------------------------------------------------ + # Serving-config guard. grpc_serve reads hyperparameters ONCE, at this + # call, for TLS, ports and root_log_dir. Serving over the network before + # config is wrapped can bring the backend up UNSECURED (TLS falls back to + # GRPC_TLS_ENABLED, default off), so hard-fail on gRPC/bore; CLI-only is + # local, so warn. Escape hatch: allow_unconfigured / WL_ALLOW_UNCONFIGURED_SERVE. + # ------------------------------------------------------------------ + if not has_serving_config(): + network_serving = serving_grpc or serving_bore + allow = allow_unconfigured or os.environ.get( + "WL_ALLOW_UNCONFIGURED_SERVE", "0" + ).lower() in ("1", "true", "yes") + base_msg = ( + "wl.serve() was called before any serving config was wrapped. Wrap " + 'it first with wl.watch_or_edit(parameters, flag="hyperparameters") ' + "so TLS, ports and root_log_dir come from your config instead of " + "falling back to environment variables / defaults." + ) + if network_serving and not allow: + raise RuntimeError( + base_msg + " Refusing to expose an unconfigured backend over " + "gRPC/bore — it may come up without TLS/auth. Pass " + "allow_unconfigured=True (or set WL_ALLOW_UNCONFIGURED_SERVE=1) " + "to override." + ) + logger.warning(base_msg) + if serving_grpc: grpc_serve(**kwargs) + # In a notebook/Colab the backend has no local Weights Studio to talk to + # (the UI runs on the user's own machine). Nudge them to open a tunnel. + if serving_grpc and not serving_bore and _running_in_notebook(): + logger.warning( + "Running in a notebook/Colab with serving_grpc=True but " + "serving_bore=False. Weights Studio runs on your OWN machine and " + "cannot reach this backend directly. Pass serving_bore=True to open " + "a tunnel and sync with the UI: wl.serve(serving_grpc=True, " + "serving_bore=True)." + ) + + bore_endpoint = None + if serving_bore: + from weightslab.tunnel import serve_bore + port = (bore_port + or kwargs.get("grpc_port") + or int(os.getenv("GRPC_BACKEND_PORT", 50051))) + if not serving_grpc: + logger.warning( + "serving_bore=True but serving_grpc=False — the tunnel will have " + "no gRPC backend to expose. Set serving_grpc=True." + ) + bore_endpoint = serve_bore(port=port) + if bore_endpoint: + print("=" * 60) + print(f" Backend exposed via bore at: {bore_endpoint}") + print(" On your local machine, run:") + print(" weightslab start") + print(f" weightslab tunnel {bore_endpoint}") + print("=" * 60) + if serving_cli: # The explicit parameter is the source of truth; drop any duplicate # spawn_client passed through kwargs so we don't pass it twice. kwargs.pop("spawn_client", None) cli_serve(spawn_client=spawn_cli_client, **kwargs) + return bore_endpoint -def keep_serving(timeout: int = None, release_gpu: bool = True) -> None: + +def keep_serving(timeout: int = None, release_gpu: bool = False) -> None: """Keep process alive while background WeightsLab services are running. Args: @@ -1123,7 +1635,8 @@ def keep_serving(timeout: int = None, release_gpu: bool = True) -> None: logger.info("Shutting down WeightsLab services.") -def signal(name: str, subscribe_to: str = None, compute_every_n_steps: int = 1, **kwargs): +def signal(name: str, subscribe_to: str = None, compute_every_n_steps: int = 1, + min_step: int = 0, **kwargs): """ Decorator that registers a custom signal function. @@ -1135,6 +1648,12 @@ def signal(name: str, subscribe_to: str = None, compute_every_n_steps: int = 1, name: Public signal name. Defaults to decorated function name. subscribe_to: Optional signal/metric name this signal reacts to. compute_every_n_steps: Frequency for dynamic computation. + min_step: Minimum training step before a subscribed (dynamic) signal + starts firing. The signal is skipped while ``current_step < + min_step``. Defaults to ``0`` (fire from the start). Useful when a + signal needs enough history to be meaningful — e.g. a loss-shape + classifier that should only run once each sample has a trajectory + (``min_step=505``). **kwargs: Additional signal metadata stored in the ledger. Returns: @@ -1144,7 +1663,8 @@ def signal(name: str, subscribe_to: str = None, compute_every_n_steps: int = 1, @wl.signal(name="blue_pixels") def compute_blue(sample, **kwargs): ... - @wl.signal(name="weighted_loss", subscribe_to="train_loss", compute_every_n_steps=10) + @wl.signal(name="weighted_loss", subscribe_to="train_loss", + compute_every_n_steps=10, min_step=505) def compute_weighted(value, sample_id, **kwargs): ... """ def decorator(func): @@ -1157,6 +1677,14 @@ def decorator(func): func._wl_signal_meta = kwargs func._wl_signal_meta['subscribe_to'] = subscribe_to func._wl_signal_meta['compute_every_n_steps'] = compute_every_n_steps + # Reactive dependency set. ``inputs=[...]`` is the unified form: the signal + # fires when ALL its inputs are present at a step, order-independently + # (subsumes subscribe_to + ingests). A single-input signal is the alias + # ``inputs=["x"]`` == ``subscribe_to="x"``. Signals that use the legacy + # ``subscribe_to`` keyword instead stay on the legacy dispatch. + _inp = kwargs.get('inputs') + func._wl_signal_meta['inputs'] = list(_inp) if _inp else None + func._wl_signal_meta['min_step'] = min_step func._wl_signal_name = reg_name return func @@ -1602,7 +2130,8 @@ def save_signals( targets: th.Tensor | np.ndarray | dict = None, preds: th.Tensor | np.ndarray | dict = None, step: int | None = None, - log: bool = False + log: bool = False, + _react: bool = True, ): """Save **per-sample** statistics to the tracked dataset. @@ -1666,8 +2195,10 @@ def save_signals( if DATAFRAME_M is None: DATAFRAME_M = get_dataframe() - # Get current model step - step = _get_step(step=step) + # Honor an explicit step (as documented); only infer from the model when it + # is omitted. Reactive backfill on the signal-worker thread depends on this — + # it logs derived values at the job's step, not the live (ahead) model step. + step = step if step is not None else _get_step() # Log if requested for each signals if log: @@ -1699,7 +2230,17 @@ def normalize(x): if x is None: return None if isinstance(x, list) and isinstance(x[0], list): - return [np.max(np.array([to_numpy(t) for t in row]), axis=0) for row in x] + rows = [[to_numpy(t) for t in row] for row in x] + # A sample can legitimately have zero instances (e.g. no tracked + # class visible in frame); fall back to an all-background mask + # shaped like its neighbors instead of reducing an empty array. + shape = next((r[0].shape for r in rows if r), None) + dtype = next((r[0].dtype for r in rows if r), np.uint16) + return [ + np.max(np.array(row), axis=0) if row + else np.zeros(shape, dtype=dtype) if shape is not None else np.zeros(0, dtype=dtype) + for row in rows + ] elif isinstance(x, list): return [to_numpy(t) for t in x] if isinstance(x, th.Tensor): @@ -1750,6 +2291,18 @@ def expand_dim(x): step=step ) + # Reactive signals: these just-logged signals may satisfy an inputs=[...] + # signal. Only logged (queryable) signals can be inputs. _react=False on the + # reactive persist path prevents re-entrant dispatch. + if _react and log and isinstance(signals, dict): + try: + _dispatch_or_enqueue(list(signals.keys()), batch_ids, step, + get_active_origin() or 'train') + except StaleSignalError: + raise + except Exception as e: + logger.debug(f"reactive dispatch after save_signals failed: {e}") + def save_instance_signals( signals: dict, @@ -3189,7 +3742,9 @@ def write_history( graph_name=None, experiment_hash: str | None = None, sample_id=None, + orient: str = "columns", instance_id=None, + verbose: bool = False, ) -> str: """Dump signal history to *path* as JSON or CSV. @@ -3229,6 +3784,14 @@ def write_history( instance_id : int or list of int, optional Restrict per-instance rows to one or more annotation IDs. Has no effect on global or per-sample history. + orient : str, optional + JSON layout for each section (``"global"``/``"sample"``/``"instance"``), + forwarded to ``pandas.DataFrame.to_json``. Default ``"columns"`` + (``{column: {row_index: value}}``) — compact, since each column name + is written once per section instead of once per row. Pass + ``"records"`` for the row-list-of-dicts shape + (``[{"graph_name": ..., "step": ..., ...}, ...]``) if that's more + convenient downstream. Ignored for ``format="csv"``. Returns ------- @@ -3253,6 +3816,10 @@ def write_history( type_of_history="sample", experiment_hash="abc123", ) + + Get the row-list-of-dicts JSON shape instead of the compact default:: + + wl.write_history("history.json", orient="records") """ import csv as _csv import json as _json @@ -3266,6 +3833,10 @@ def write_history( sample_id, instance_id, ) + # Progress lines are INFO only when verbose; otherwise DEBUG (quiet by + # default so periodic exports don't spam a training log). + _log = logger.info if verbose else logger.debug + _lg = get_logger() if _lg is None: logger.warning( @@ -3287,7 +3858,7 @@ def write_history( logger.debug("write_history: failed to resolve root_log_dir (%s); " "falling back to current directory.", _e) path = "." - logger.info("write_history: no path given, using output directory %r.", path) + _log("write_history: no path given, using output directory %r.", path) fmt = format.lower().strip() @@ -3332,7 +3903,7 @@ def write_history( write_sample = _type in ("all", "sample") write_instance = _type in ("all", "instance") - logger.info( + _log( "write_history: resolved filters → type=%r, experiment_hash=%s, " "graph_name=%s, sample_id=%s, instance_id=%s", _type, @@ -3357,10 +3928,10 @@ def write_history( ) _phash = _hashlib.md5(str(_params_key).encode()).hexdigest()[:8] path = _os.path.join(path, f"{_phash}_history.{fmt}") - logger.info("write_history: auto-generated filename %r (params hash " - "%s).", _os.path.basename(path), _phash) + _log("write_history: auto-generated filename %r (params hash " + "%s).", _os.path.basename(path), _phash) _os.makedirs(_os.path.dirname(_os.path.abspath(path)), exist_ok=True) - logger.info("write_history: output file → %s", _os.path.abspath(path)) + _log("write_history: output file → %s", _os.path.abspath(path)) global_rows: list = [] sample_rows: list = [] @@ -3448,8 +4019,20 @@ def write_history( payload["sample"] = sample_rows if write_instance: payload["instance"] = instance_rows + + # Each section is a list of row dicts sharing the same keys; route it + # through pandas so `orient` is genuinely honored. Default "columns" + # ({column: {row_index: value}}) writes each column name once per + # section instead of once per row — much smaller for many rows. + # orient="records" reproduces the original row-list-of-dicts shape. + import pandas as _pd + json_payload = { + section: _json.loads(_pd.DataFrame(rows).to_json(orient=orient, default_handler=str)) + for section, rows in payload.items() + } + with open(path, "w", encoding="utf-8") as fh: - _json.dump(payload, fh, indent=2) + _json.dump(json_payload, fh, indent=2) elif fmt == "csv": _CSV_FIELDS = [ @@ -3474,7 +4057,7 @@ def write_history( ) _total = len(global_rows) + len(sample_rows) + len(instance_rows) - logger.info( + _log( "write_history: wrote %d row(s) (global=%d, sample=%d, instance=%d) " "as %s to %s", _total, len(global_rows), len(sample_rows), len(instance_rows), @@ -3491,15 +4074,180 @@ def write_history( return path +# Signal names registered via watch_or_edit(..., flag="loss") — populated +# automatically (see the loss branch above), so the background flush thread +# can auto-classify every per-sample loss without the user calling anything. +_AUTO_LOSS_SHAPE_SIGNALS: set = set() + + +def auto_loss_shape_signal_names() -> list: + """Every signal name currently registered via ``flag="loss"`` — the + automatic loss-shape classification target set. Used internally by + :class:`weightslab.backend.logger.LoggerQueue`'s background flush thread; + read it yourself only for introspection/debugging.""" + return sorted(_AUTO_LOSS_SHAPE_SIGNALS) + + +LOSS_SHAPES = ["monotonic", "plateaued", "Flat_high", "high_variance", "U_Shape", "Spiked"] + + +def trajectory_stats(values): + """Scale-invariant summary stats of one sample's trajectory — the reusable + feature layer. Returns a dict (or ``None`` with < 2 points) so you can build + a custom classifier on top without re-deriving the features:: + + def my_clf(v): + s = wl.trajectory_stats(v) + return None if s is None else ("fast" if s["drop"] > 0.6 else "slow") + """ + y = np.asarray(values, dtype=float) + if y.size < 2: + return None + n = y.size + rng = max(float(y.max() - y.min()), 1e-8) + tail = y[int(0.6 * n):] + return { + "n": n, + "first": float(y[0]), "last": float(y[-1]), + "drop": (float(y[0]) - float(y[-1])) / (abs(float(y[0])) + 1e-8), + "cv": float(y.std()) / (abs(float(y.mean())) + 1e-8), + "argmin_frac": float(np.argmin(y)) / n, + "rebound": (float(y[-1]) - float(y.min())) / rng, + "max_up_jump": float(np.diff(y).max()) / rng, + "tail_cv": float(tail.std()) / (abs(float(tail.mean())) + 1e-8), + } + + +def classify_loss_shape(values, min_points=5, drop_learned=0.4, drop_plateau=0.15, + tail_flat_cv=0.1, spike_jump=0.5, noisy_cv=0.5, u_rebound=0.3): + """Default classifier -> one of ``LOSS_SHAPES`` (or ``None`` with < + *min_points*). Built on :func:`trajectory_stats` (reuse those features + for your own rules) with every threshold a named, tunable param. Override the + whole thing by passing your own ``classifier`` to :func:`write_signal_shapes`.""" + s = trajectory_stats(values) + if s is None or s["n"] < min_points: + return None + if 0.2 < s["argmin_frac"] < 0.8 and s["rebound"] > u_rebound: + return "U_Shape" + if s["drop"] > drop_learned: + return "monotonic" + if s["drop"] > drop_plateau and s["tail_cv"] < tail_flat_cv: + return "plateaued" + if s["max_up_jump"] > spike_jump: + return "Spiked" + if s["cv"] > noisy_cv: + return "high_variance" + return "Flat_high" + + +def write_signal_shapes(signal_name, tag_name=None, classifier=None): + """Reusable engine: classify every sample's trajectory of *signal_name* into + a categorical tag and return the ``{label: count}`` distribution. Works for + ANY per-sample signal — loss, accuracy, a second loss, any metric. Reads the + full history once (cheap end-of-report reduction). *classifier* (trajectory + -> label|None) defaults to the loss-shaped one (for a decreasing metric); + pass your own for e.g. accuracy (increasing). *tag_name* defaults to + ``'_shape'``.""" + clf = classifier or classify_loss_shape + if tag_name is None: + tag_name = signal_name + "_shape" if signal_name.endswith('_loss') else signal_name + "_loss_shape" + series = {} + for sid, step, val, _ in query_signal_history(signal_name): + series.setdefault(sid, []).append((step, val)) + by_label = {} + for sid, pts in series.items(): + label = clf([v for _, v in sorted(pts)]) + if label is not None: + by_label.setdefault(label, []).append(sid) + for label, sids in by_label.items(): + set_categorical_tag(sids, tag_name, label) + return {k: len(v) for k, v in by_label.items()} + + +def write_loss_shapes(loss_signal="loss_sample", classifier=None): + """Convenience wrapper over :func:`write_signal_shapes` for the loss signal + (tag ``loss_shape``, default loss classifier).""" + return write_signal_shapes(loss_signal, tag_name="loss_shape", classifier=classifier) + + +def enable_loss_shape_autotag(loss_signal=None, tag_name=None, classifier=None): + """Customize automatic loss-shape tagging — **not required to turn it on**. + + Every signal registered via ``wl.watch_or_edit(criterion, flag="loss", ...)`` + is already auto-classified in the background with zero setup: the logger's + periodic flush thread (``WL_LOGGER_FLUSH_INTERVAL_SECONDS`` env var, default + 2s) discovers it automatically and re-tags it as ``'_shape'`` every + tick, once it has enough per-sample history to classify (see + :func:`classify_loss_shape`'s ``min_points``) — no call needed, and no + ``write_dataframe(loss_shape_signal=...)`` required either. + + Call this only to override the tag name or classifier used for one specific + *loss_signal* (e.g. it isn't a decreasing loss, so the default classifier is + wrong for it). Re-enables that signal if it was previously disabled via + :func:`disable_loss_shape_autotag`. + """ + if loss_signal is None: + raise ValueError( + "enable_loss_shape_autotag: loss_signal is required — every " + "flag='loss' signal is already auto-classified without calling " + "this; only call it to override the tag_name/classifier for one." + ) + lg = get_logger() + if lg is None: + raise RuntimeError( + "enable_loss_shape_autotag: no logger registered yet — call this " + "after wl.watch_or_edit(criterion, flag='loss', log=True)." + ) + lg.set_loss_shape_override(loss_signal, tag_name=tag_name, classifier=classifier) + + +def disable_loss_shape_autotag(loss_signal=None): + """Stop automatic loss-shape tagging for *loss_signal*, or for every + signal (including ones registered later) if *loss_signal* is ``None``.""" + lg = get_logger() + if lg is not None: + lg.disable_loss_shape_autotag(loss_signal) + + +def enable_loss_shape_signal(loss_signal="loss_sample", name="sig/loss_shape", + every=1, classifier=None): + """Register a LIVE per-step ``@wl.signal`` that classifies each sample's + loss-trajectory-so-far into an int-coded shape (index into ``LOSS_SHAPES``, + ``-1`` before there's enough history), updated every *every* steps. + + This is the live counterpart to :func:`write_loss_shapes` (report-time). It's + heavier — it reads history on every fire — so throttle with *every* or prefer + the report-time path for a definitive, full-coverage tag.""" + clf = classifier or classify_loss_shape + + @signal(name=name, inputs=[loss_signal], batched=True, compute_every_n_steps=every) + def _live_shape(b): + hist = b.history(loss_signal) + return np.array([(LOSS_SHAPES.index(lbl) if (lbl := clf(hist[s])) is not None else -1) + for s in b.sample_ids], dtype=float) + return _live_shape + + def write_dataframe( path: str | None = None, format: str = "json", columns=None, sample_id=None, instance_id=None, + orient: str = "columns", + verbose: bool = False, + loss_shape_signal: str | None = None, ) -> str: """Dump the WeightsLab sample dataframe to *path* as JSON or CSV. + Every ``flag="loss"`` signal is already auto-classified into a + ``'_shape'`` tag in the background with zero setup (see + :func:`enable_loss_shape_autotag`'s docstring), so you normally don't need + *loss_shape_signal* at all. It remains as a one-off: when set (e.g. + ``"loss_sample"``), the classifier runs synchronously first so this + specific report is guaranteed fresh at the moment it's written, rather than + whatever the background thread last computed. + Parameters ---------- path : str, optional @@ -3567,12 +4315,27 @@ def write_dataframe( import os as _os import hashlib as _hashlib + # If reactive signals run on the worker thread, process every queued job + # before reading, so the report includes the latest steps' derived signals. + drain_signals() + + # Default report summary: tag each sample's loss shape + log the distribution. + if loss_shape_signal is not None: + try: + dist = write_loss_shapes(loss_shape_signal) + logger.info("[WeightsLab] loss-shape summary (%s): %s", loss_shape_signal, dist) + except Exception as e: + logger.debug("loss-shape summary skipped: %s", e) + logger.debug( "write_dataframe called: path=%r, format=%r, columns=%r, " "sample_id=%r, instance_id=%r", path, format, columns, sample_id, instance_id, ) + # INFO only when verbose; DEBUG otherwise (quiet periodic exports). + _log = logger.info if verbose else logger.debug + _dm = get_dataframe() if _dm is None: logger.warning( @@ -3592,7 +4355,7 @@ def write_dataframe( logger.debug("write_dataframe: failed to resolve root_log_dir (%s); " "falling back to current directory.", _e) path = "." - logger.info("write_dataframe: no path given, using output directory %r.", path) + _log("write_dataframe: no path given, using output directory %r.", path) fmt = format.lower().strip() @@ -3611,7 +4374,7 @@ def write_dataframe( if columns is not None and not (isinstance(columns, str) and columns.lower() == "all"): _col_filter = [columns] if isinstance(columns, str) else list(columns) - logger.info( + _log( "write_dataframe: resolved filters → columns=%s, sample_id=%s, instance_id=%s", _col_filter if _col_filter is not None else "", _sid_filter if _sid_filter is not None else "", @@ -3629,10 +4392,10 @@ def write_dataframe( ) _phash = _hashlib.md5(str(_params_key).encode()).hexdigest()[:8] path = _os.path.join(path, f"{_phash}_dataframe.{fmt}") - logger.info("write_dataframe: auto-generated filename %r (params hash %s).", - _os.path.basename(path), _phash) + _log("write_dataframe: auto-generated filename %r (params hash %s).", + _os.path.basename(path), _phash) _os.makedirs(_os.path.dirname(_os.path.abspath(path)), exist_ok=True) - logger.info("write_dataframe: output file → %s", _os.path.abspath(path)) + _log("write_dataframe: output file → %s", _os.path.abspath(path)) # Flush pending buffer to H5 before reading try: @@ -3703,7 +4466,7 @@ def write_dataframe( df_out = df_out.reset_index() if fmt == "json": - _json_str = df_out.to_json(orient="records", default_handler=str) + _json_str = df_out.to_json(orient=orient, default_handler=str) with open(path, "w", encoding="utf-8") as fh: _json.dump(_json.loads(_json_str), fh, indent=2) diff --git a/weightslab/trainer/experiment_context.py b/weightslab/trainer/experiment_context.py index 36dccc86..082d65a4 100644 --- a/weightslab/trainer/experiment_context.py +++ b/weightslab/trainer/experiment_context.py @@ -184,7 +184,7 @@ def _get_last_hash(): last_hash = "000000000000000000000000" try: chkpt_m = self._components.get("checkpoint_manager") - if chkpt_m and hasattr(chkpt_m, "get_last_checkpoint_hash"): + if chkpt_m and hasattr(chkpt_m, "get_current_experiment_hash"): resolved = chkpt_m.get_current_experiment_hash() if resolved: last_hash = resolved diff --git a/weightslab/trainer/services/agent/agent.py b/weightslab/trainer/services/agent/agent.py index dcede2c7..08aaa572 100644 --- a/weightslab/trainer/services/agent/agent.py +++ b/weightslab/trainer/services/agent/agent.py @@ -731,6 +731,13 @@ def _load_config(self): self.openrouter_base_url = os.environ.get("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1") self.openrouter_api_key = os.environ.get("OPENROUTER_API_KEY", None) self.openrouter_request_timeout = float(os.environ.get("OPENROUTER_REQUEST_TIMEOUT", "15.0")) + # Cap the completion length. OpenRouter pre-authorizes + # `max_tokens * completion_price` against the key's remaining budget + # BEFORE generating; with no cap the client requests the model's full + # output window (tens of thousands of tokens), which 402s on a + # credit/weekly-limited key even though the actual intent-planning + # response is only a few hundred tokens. Keep this modest. + self.openrouter_max_tokens = int(os.environ.get("OPENROUTER_MAX_TOKENS", "2048")) # Bias OpenRouter's upstream provider selection ("throughput"/"latency"/ # "price"); empty string lets OpenRouter choose freely (see Phase B.2). self.openrouter_provider_sort = os.environ.get("OPENROUTER_PROVIDER_SORT", "throughput") @@ -783,6 +790,7 @@ def _load_config(self): self.openrouter_base_url = a_cfg.get("openrouter_base_url", self.openrouter_base_url) self.openrouter_api_key = a_cfg.get("openrouter_api_key", self.openrouter_api_key) self.openrouter_request_timeout = float(a_cfg.get("openrouter_request_timeout", self.openrouter_request_timeout)) + self.openrouter_max_tokens = int(a_cfg.get("openrouter_max_tokens", self.openrouter_max_tokens)) self.openrouter_provider_sort = a_cfg.get("openrouter_provider_sort", self.openrouter_provider_sort) self.openrouter_structured_output = bool(a_cfg.get("openrouter_structured_output", self.openrouter_structured_output)) @@ -872,6 +880,7 @@ def _setup_providers(self): api_key=self.openrouter_api_key, base_url=openrouter_base_url, streaming=False, max_retries=1, request_timeout=self.openrouter_request_timeout, + max_tokens=self.openrouter_max_tokens, extra_body=extra_body, ) self.chain_openrouter = llm @@ -930,7 +939,17 @@ def _check_chat_provider(self, provider: str) -> "tuple[bool, str]": return False, f"{provider} client was not initialized." try: - response = chain.invoke("Reply with OK.") + # Keep the probe's reservation tiny. OpenRouter pre-authorizes + # `max_tokens * price` before generating, so a large (or unset) + # cap makes the health check 402 on a budget-limited key even + # though the model is perfectly usable for real (short) requests. + probe = chain + if provider == "openrouter" and hasattr(chain, "bind"): + try: + probe = chain.bind(max_tokens=16) + except Exception: + probe = chain + response = probe.invoke("Reply with OK.") except Exception as e: _LOGGER.warning(f"[{provider}] connectivity check failed: {e}") return False, f"{provider} connectivity check failed: {e}" diff --git a/weightslab/trainer/services/agent/intent_prompt.py b/weightslab/trainer/services/agent/intent_prompt.py index a9fe7bba..663a91d3 100644 --- a/weightslab/trainer/services/agent/intent_prompt.py +++ b/weightslab/trainer/services/agent/intent_prompt.py @@ -74,13 +74,10 @@ - "average train loss over training below 0.2" → `signal_history('train_loss','mean') < 0.2` Use it exactly like a normal column expression (create a tag, discard, or keep on it — see Ex39). A sample with no recorded history yields NaN (comparisons are False), so it's safely excluded. 11. **Hyperparameter tuning (`action_name="set_hyperparam"`)**: to change a training hyperparameter, emit a `kind="action"` step with `action_name="set_hyperparam"` and `action_params={{"param": , "op": <"set"|"scale">, "value": }}`. - - `param`: prefer a semantic name — `"batch_size"`, `"learning_rate"`, `"dump_ratio"` (the model-dump/checkpoint ratio), `"eval_ratio"` (the evaluation ratio) — or an exact dotted config path (e.g. `"data.train_loader.batch_size"`). The backend resolves the semantic name to the real config key. - - Absolute change ("set X to N", "change X to N") → `op="set"`, `value=N`. - Relative change ("increase X by 10%", "decrease by 20%") → `op="scale"`, `value` = the multiplier (10% increase → `1.1`; 20% decrease → `0.8`). Compute the multiplier yourself. - See Ex45–Ex48. --- -## 4. SCHEMA RULES (STRICT) - **Primary Goal**: `ui_manipulation` (grid changes), `data_analysis` (answers), `action` (external), `model_management` (model_info/model_action), `out_of_scope`. - **Atomic Operations**: - `conditions`: List of dicts with keys "column", "op", "value". Operators: `==, !=, >, <, >=, <=, contains, in, max, min`. diff --git a/weightslab/trainer/services/data_image_utils.py b/weightslab/trainer/services/data_image_utils.py index 81358b32..30695b83 100644 --- a/weightslab/trainer/services/data_image_utils.py +++ b/weightslab/trainer/services/data_image_utils.py @@ -165,6 +165,63 @@ def encode_image_webp(pil_image, quality=70, method=2): return b"" +# ============================================================================= +# Tabular inputs (1-D feature vectors) +# ============================================================================= +# Tabular samples have no image — the model input IS a 1-D feature vector. We +# transmit the actual feature values losslessly in the DataStat ``value`` field +# (type ``"vector"``), and attach a small heatmap so image-only grids still show +# a cell. The List Exploration view reads the per-feature metadata columns; a +# tabular-aware grid can read ``value`` directly. + +def looks_like_tabular(np_img) -> bool: + """True when the sample input is a 1-D feature vector (no spatial dims).""" + return np_img is not None and getattr(np_img, "ndim", None) == 1 + + +def render_tabular_heatmap(vec: np.ndarray, cell: int = 16, quality: int = 60): + """Render a 1-D feature vector as a small square grayscale heatmap. + + Returns ``(webp_bytes, [h, w, c])``. Values are min-max normalised for + display only; the exact values travel in the DataStat ``value`` field. + """ + v = np.asarray(vec, dtype=np.float32).ravel() + n = int(v.size) + if n == 0: + return b"", [0, 0, 0] + side = int(np.ceil(np.sqrt(n))) + padded = np.zeros(side * side, dtype=np.float32) + padded[:n] = v + grid = padded.reshape(side, side) + mn, mx = float(np.nanmin(grid)), float(np.nanmax(grid)) + norm = (grid - mn) / (mx - mn + 1e-8) + img8 = np.clip(norm * 255.0, 0, 255).astype(np.uint8) + try: + pil = Image.fromarray(img8, mode="L").resize( + (side * cell, side * cell), Image.Resampling.NEAREST) + buf = io.BytesIO() + pil.convert("RGB").save(buf, format="WEBP", quality=quality, method=1) + data = buf.getvalue() + buf.close() + return data, [side * cell, side * cell, 3] + except Exception as e: # pragma: no cover - display-only fallback + logger.warning(f"Failed to render tabular heatmap: {e}") + return b"", [side, side, 1] + + +def build_tabular_raw_data_stat(vec: np.ndarray): + """Build the ``raw_data`` DataStat for a tabular (1-D) input. + + The feature values are carried losslessly in ``value`` (type ``"vector"``, + ``shape=[N]``); a heatmap rides along in ``thumbnail`` for display only. + """ + v = np.asarray(vec, dtype=np.float32).ravel() + values = [float(x) for x in v] + thumb, _shape = render_tabular_heatmap(v) + return create_data_stat( + "raw_data", "vector", shape=[len(values)], value=values, thumbnail=thumb) + + def resize_mask_nearest(mask_arr: np.ndarray, target_w: int, target_h: int) -> np.ndarray: """Resize a 2D+ mask array using nearest-neighbour (preserves class IDs). diff --git a/weightslab/trainer/services/data_service.py b/weightslab/trainer/services/data_service.py index 454e2a12..a5b3923e 100755 --- a/weightslab/trainer/services/data_service.py +++ b/weightslab/trainer/services/data_service.py @@ -46,6 +46,8 @@ rle_encode_mask, create_data_stat, resize_mask_nearest, + looks_like_tabular, + build_tabular_raw_data_stat, ) @@ -81,6 +83,29 @@ def _point_cloud_chunk_bytes() -> int: return val +def peek_is_tabular_sample(dataset, sample_id) -> bool: + """True when ``sample_id``'s raw model input is a 1-D feature vector. + + Used as the last step of task_type auto-detection: image/detection/ + segmentation datasets are identified by their *label* shape, but a tabular + dataset's label is a plain scalar, so the only way to tell it apart is to + peek at the *input* it feeds the model. Any resolution failure (missing + sample, dataset without the expected lookup methods, …) is treated as + "not tabular" rather than raised, since this is a best-effort heuristic. + """ + try: + if hasattr(dataset, "get_physical_location"): + idx, rank = dataset.get_physical_location(sample_id) + else: + idx, rank = dataset.get_index_from_sample_id(sample_id), 0 + if idx is None: + return False + img, _, _, _ = load_raw_image_array(dataset, idx, rank=rank) + return looks_like_tabular(img) + except (KeyError, ValueError, AttributeError, IndexError): + return False + + def normalize_metadata_copy_source_name(source_name: str, experiment_hash: str = None) -> str: """Normalize a source metadata name for deterministic copied-column naming.""" name = str(source_name or "").strip() @@ -493,7 +518,12 @@ def _build_preview_cache(self) -> None: ds_idx = dataset.get_index_from_sample_id(sample_id) member_rank = 0 - _, _, _, pil_img = load_raw_image_array(dataset, ds_idx, rank=member_rank) + _pv_np_img, _, _, pil_img = load_raw_image_array(dataset, ds_idx, rank=member_rank) + if looks_like_tabular(_pv_np_img): + # Tabular sample: cache the feature values (lossless) as a + # 'vector' raw_data stat + heatmap thumbnail, not an image. + stats.append(build_tabular_raw_data_stat(_pv_np_img)) + pil_img = None # skip the image-thumbnail path below if pil_img is not None: ar = pil_img.size[0] / max(pil_img.size[1], 1) if PREVIEW_SIZE >= max(pil_img.size): @@ -595,6 +625,10 @@ def _build_preview_cache(self) -> None: task_type = str(db_task).strip().lower() if db_task and str(db_task).strip().lower() not in ("none", "nan", "unknown", "") else "unknown" if task_type == "unknown" and _early_task: task_type = _early_task + if task_type == "unknown" and looks_like_tabular(_pv_np_img): + # Tabular: the model INPUT (not the label) is a 1-D feature + # vector — no spatial dims to key a heuristic off of. + task_type = "tabular" if task_type == "unknown" and label is not None: l_arr = to_numpy_safe(label) if l_arr is not None and l_arr.ndim >= 2 and l_arr.shape[-2] >= 16 and l_arr.shape[-1] >= 16: @@ -860,8 +894,10 @@ def _pull_into_all_data_view_df(self): # The manager now expands samples into one row per (sample_id, annotation_id) # instance. Collapse back to one row per sample for the sample-centric UI/agent - # view, nesting per-instance signals into a dict column. - df = self._df_manager.get_collapse_annotations_to_samples_df() + # view, nesting per-instance signals into a dict column. Reuse the frame we just + # pulled — otherwise collapse would re-run get_combined_df() (full copy + buffer + # merge + proxy conversion) a second time over the whole dataset every refresh. + df = self._df_manager.get_collapse_annotations_to_samples_df(df) # Ensure sample_id is a column if it was the index df = safe_reset_index(df) @@ -1143,10 +1179,20 @@ def process_sample(args): if "origin" in df_update.columns: del df_update["origin"] - self._df_manager.upsert_df(df_update, origin=origin, force_flush=True) + try: + self._df_manager.upsert_df(df_update, origin=origin, force_flush=True) + except Exception as e: + logger.warning( + "[DataService] Skipping natural sort stats write for origin=%s: %s", + origin, + e, + ) # Force refresh internal view - self._slowUpdateInternals(force=True) + try: + self._slowUpdateInternals(force=True) + except Exception as e: + logger.warning("[DataService] Natural sort refresh failed, continuing startup: %s", e) logger.info(f"[DataService] Completed stats computation for {processed_count} samples") print(f"\n\nNatural sort computation finished for {processed_count} samples\n\n") @@ -1285,6 +1331,13 @@ def _process_sample_row(self, args): except Exception: pass + # Tabular: the model INPUT (not the label) is a 1-D feature vector. + # Only peek at it when nothing else already matched, so this is a + # cheap no-op for image/detection/segmentation datasets. + if task_type == "classification" and dataset is not None: + if peek_is_tabular_sample(dataset, sample_id): + task_type = "tabular" + # ====== Step 5a: Metadata stats — moved to GetMetaData ====== # Generic dataframe metadata columns (signals, tags, custom fields, etc.) # are no longer returned by GetDataSamples; the dedicated GetMetaData RPC @@ -1323,7 +1376,7 @@ def _process_sample_row(self, args): # ====== Step 7: Process labels ====== t_label_convert = 0.0 - if not skip_label_for_request and task_type != "classification": + if not skip_label_for_request and task_type not in ("classification", "tabular"): t0_gt_conv = time.time() label_raw = row.get(SampleStatsEx.TARGET.value) if label is None else label if label_raw is None and dataset is not None: @@ -1581,7 +1634,7 @@ def _json_default(o): if skip_prediction_for_request: pred = None - if task_type != "classification" and pred is not None: + if task_type not in ("classification", "tabular") and pred is not None: t0_pmask = time.time() # 3D (point cloud) detection predictions: same dual payload as @@ -1748,7 +1801,12 @@ def _json_default(o): else: np_img, is_volumetric, original_shape, middle_pil = None, False, [], None - if middle_pil is not None: + # Tabular input: the model input is a 1-D feature vector, not an + # image. Transmit the actual values (lossless) as a 'vector' + # raw_data stat instead of a lossy WebP image. + if looks_like_tabular(np_img): + data_stats.append(build_tabular_raw_data_stat(np_img)) + elif middle_pil is not None: original_size = middle_pil.size target_width = original_size[0] target_height = original_size[1] @@ -2653,6 +2711,45 @@ def _agent_show_config(self, param=None) -> str: text = text[:max_len] + "\n... (truncated; ask for a specific key, e.g. 'show the root log dir')" return f"Configuration ({hp_name}):\n{text}" + @staticmethod + def _mask_from_coerced_query(df, expr: str): + """Best-effort boolean mask for `expr` against `df`, tolerating two + shapes of "numeric in practice, awkward in dtype" columns that break a + plain comparison in df.query()/df.eval(): + + 1. A column referenced by `expr` is index-level-only (e.g. ``sample_id`` + on a (sample_id, annotation_id) MultiIndex) — promoted to a real + column via the same ``safe_reset_index`` GetHistogram already uses, + so pandas' query engine can reference it at all instead of raising + obscure internal errors (e.g. "'UnaryOp' object has no attribute + 'evaluate'"). + 2. An object-dtype column holds numeric values mixed with a few + non-numeric placeholders (e.g. a per-sample signal not yet computed + for every row) — cast via ``pd.to_numeric`` (errors="coerce"), + matching GetHistogram's own is_numeric classification, so a filter + the UI offered as numeric doesn't crash with e.g. "'<' not + supported between instances of 'str' and 'int'". + + Returns a boolean array positionally aligned to `df` (safe to apply as + ``df[mask]`` regardless of `df`'s actual index), or None if `expr` + doesn't parse/evaluate even against this coerced view. + """ + reset = safe_reset_index(df) + candidate_names = set(re.findall(r'`([^`]+)`', expr)) | set(re.findall(r'\b[A-Za-z_]\w*\b', expr)) + to_coerce = {} + for col in candidate_names: + if col not in reset.columns or reset[col].dtype != object: + continue + numeric_vals = pd.to_numeric(reset[col], errors="coerce") + if numeric_vals.notna().any(): + to_coerce[col] = numeric_vals + coerced = reset.assign(**to_coerce) if to_coerce else reset + try: + mask = coerced.eval(expr, local_dict={"df": coerced, "np": np, "pd": pd}) + except Exception: + return None + return np.asarray(mask, dtype=bool) + def _apply_agent_operation(self, df, func: str, params: dict) -> str: """ Apply an agent-described operation to df in-place. @@ -2762,8 +2859,23 @@ def _apply_agent_operation(self, df, func: str, params: dict) -> str: df.drop(index=df.index.difference(kept.index), inplace=True) return f"Applied query (eval): {expr}" except Exception as eval_e: - logger.error(f"Query failed: {e} | Eval failed: {eval_e}") - return f"Failed to filter: {expr}" + # Both a plain df.query and df.eval can fail on a column that + # is "numeric enough" by GetHistogram's own classification but + # awkward in practice — either index-level-only (e.g. + # sample_id on a MultiIndex) or object-dtype with a stray + # non-numeric placeholder among otherwise-numeric values (e.g. + # a per-sample signal not yet computed for every row). Retry + # once against a coerced view (see _mask_from_coerced_query) + # so a filter the UI offered as numeric doesn't fail here with + # e.g. "'<' not supported between instances of 'str' and + # 'int'". + mask = self._mask_from_coerced_query(df, expr) + if mask is None: + logger.error(f"Query failed: {e} | Eval failed: {eval_e}") + return f"Failed to filter: {expr}" + kept = df[mask] + df.drop(index=df.index.difference(kept.index), inplace=True) + return f"Applied query: {expr}" # C) Column Modification (Transform) if func == "df.modify": @@ -3429,6 +3541,48 @@ def _is_metadata_only_request(self, request) -> bool: except Exception: return False + @staticmethod + def _downsample_curve(vals, max_points=32): + """Uniformly downsample a series to <= max_points floats (endpoints kept).""" + if len(vals) <= max_points: + return [float(v) for v in vals] + idx = np.linspace(0, len(vals) - 1, max_points).round().astype(int) + return [float(vals[j]) for j in idx] + + def _loss_trajectory_curves(self, sample_ids): + """Downsampled per-sample loss curve for the on-cell sparkline. + + Returns ``{str(sample_id): [float, ...]}`` for samples with history; an + empty dict when there is no logger or no per-sample loss signal. Read-only — + shape classification lives on the write path (enable_loss_shape_signal), + not here. Signal name overridable via WL_LOSS_TRAJ_SIGNAL (default "loss"). + """ + from weightslab.backend import ledgers + + logger_q = ledgers.get_logger() + if logger_q is None: + return {} + + signal = os.environ.get("WL_LOSS_TRAJ_SIGNAL", "loss") + try: + rows = logger_q.query_per_sample(signal, sample_ids=sample_ids) + except Exception: + logger.debug("loss_trajectory query skipped", exc_info=True) + return {} + + by_sid = {} + for sid, step, val, _ in rows: + by_sid.setdefault(str(sid), []).append((int(step), float(val))) + + curves = { + sid: self._downsample_curve([v for _, v in sorted(pts)]) + for sid, pts in by_sid.items() + } + if curves: + logger.info("[loss_traj] page_ids=%d with_history=%d signal=%s", + len(sample_ids), len(curves), signal) + return curves + def _build_metadata_only_response(self, df_slice: pd.DataFrame, requested_cols=None): """Build a DataSamplesResponse of metadata DataRecords from dataframe columns only. @@ -3547,6 +3701,20 @@ def _build_metadata_only_response(self, df_slice: pd.DataFrame, requested_cols=N _DataStat(name=col, type="string", shape=[1], value_string="1") ) + # -- Per-sample loss trajectory (the on-cell sparkline; read-only) ----- + # Ship each visible sample's loss curve as an 'array' DataStat for the grid + # sparkline. The loss-SHAPE haze is NOT computed here: classification runs on + # the write/train path (enable_loss_shape_signal / write_loss_shapes) which + # maintains the tag:loss_shape column, so this path does no per-request work. + loss_curves = self._loss_trajectory_curves(sample_ids) + for i, sid in enumerate(sample_ids): + curve = loss_curves.get(str(sid)) + if not curve: + continue + row_stats[i].append( + _DataStat(name="loss_trajectory", type="array", + shape=[len(curve)], value=curve)) + # -- Build DataRecord list in one comprehension ----------------------- _DataRecord = pb2.DataRecord data_records = [ @@ -4213,6 +4381,12 @@ def status_cb(msg: str): if isinstance(operations, dict): operations = [operations] # Backwards compat if not operations: operations = [] + # Cooperative cancel: if the client aborted during the (possibly + # long) LLM call, skip executing the operations entirely. + if abort_event.is_set(): + logger.info("[ApplyDataQuery] Canceled by client before execution") + return pb2.DataQueryResponse(success=False, message="Query canceled.") + with self._lock: self._slowUpdateInternals(force=True) @@ -4228,6 +4402,11 @@ def status_cb(msg: str): _SCHEMA_MUTATING_FUNCS = {"df.modify", "df.drop_column"} for op in operations: + # Cooperative cancel between operations (frees the lock + # promptly when the client aborts a multi-step query). + if abort_event.is_set(): + logger.info("[ApplyDataQuery] Canceled by client mid-execution") + return pb2.DataQueryResponse(success=False, message="Query canceled.") func = op.get("function") params = op.get("params", {}) or {} diff --git a/weightslab/trainer/services/experiment_service.py b/weightslab/trainer/services/experiment_service.py index 2a4a66d7..b7313c4e 100644 --- a/weightslab/trainer/services/experiment_service.py +++ b/weightslab/trainer/services/experiment_service.py @@ -727,9 +727,28 @@ def _handle_save_checkpoint(self, save_op, components, context): self._log_audit("checkpoint_save", "success", audit_details) return pb2.CommandResponse(success=True, message=msg) + def _handle_restart_instance(self): + self._log_audit("restart_instance", "success", {}) + logger.warning( + "[ExperimentCommand] Restart requested via UI; exiting process so the " + "container's restart policy brings it back up and reloads the last checkpoint." + ) + + def _delayed_exit(): + # Give the gRPC response a moment to flush to the client before the + # process dies. + time.sleep(0.5) + os._exit(0) + + threading.Thread(target=_delayed_exit, daemon=True).start() + return pb2.CommandResponse(success=True, message="Restart requested. The instance will restart shortly.") + # Training & hyperparameter commands # ------------------------------------------------------------------------- def ExperimentCommand(self, request, context): + if request.HasField("restart_operation"): + return self._handle_restart_instance() + self._ctx.ensure_components() components = self._ctx.components @@ -773,11 +792,11 @@ def ExperimentCommand(self, request, context): checkpoint_manager = ledgers.get_checkpoint_manager() except Exception: checkpoint_manager = None - if checkpoint_manager is not None and hasattr(checkpoint_manager, "save_logger_snapshot"): + if checkpoint_manager is not None and hasattr(checkpoint_manager, "flush_logger_to_disk"): try: - checkpoint_manager.save_logger_snapshot() + checkpoint_manager.flush_logger_to_disk() except Exception: - logger.debug("Could not persist logger snapshot after note update", exc_info=True) + logger.debug("Could not persist logger history after note update", exc_info=True) # Log to audit trail self._log_audit( diff --git a/weightslab/trainer/trainer_services.py b/weightslab/trainer/trainer_services.py index 0400c434..030b31e7 100644 --- a/weightslab/trainer/trainer_services.py +++ b/weightslab/trainer/trainer_services.py @@ -32,6 +32,25 @@ def _is_truthy(value: str | None) -> bool: return str(value or "").strip().lower() in {"1", "true", "yes", "on"} +_WEIGHTSLAB_VERSION: str | None = None + + +def _weightslab_version() -> str: + """Backend weightslab package version, resolved once and cached. + + Sent to the UI (via GetLatestLoggerDataResponse) so the studio can show the + backend version alongside the UI version in the logo tooltip. + """ + global _WEIGHTSLAB_VERSION + if _WEIGHTSLAB_VERSION is None: + try: + from weightslab import __version__ as v + _WEIGHTSLAB_VERSION = str(v or "") + except Exception: + _WEIGHTSLAB_VERSION = "" + return _WEIGHTSLAB_VERSION + + def _load_auth_tokens() -> set[str]: """Load accepted bearer/API-key tokens from env vars.""" raw_values: list[str] = [] @@ -408,7 +427,14 @@ def ResetAgent(self, request, context): # ------------------------------------------------------------------------- def GetLatestLoggerData(self, request, context): logger.debug(f"ExperimentServiceServicer.GetLatestLoggerData({request})") - return self._exp_service.GetLatestLoggerData(request, context) + response = self._exp_service.GetLatestLoggerData(request, context) + # Advertise the backend version to the UI (shown in the logo tooltip + # next to the UI version). Best-effort: never fail the poll over this. + try: + response.weightslab_version = _weightslab_version() + except Exception: + pass + return response # ------------------------------------------------------------------------- # Training & hyperparameter commands diff --git a/weightslab/tunnel.py b/weightslab/tunnel.py new file mode 100644 index 00000000..ba10313c --- /dev/null +++ b/weightslab/tunnel.py @@ -0,0 +1,350 @@ +"""Raw-TCP tunnel client for Weights Studio. + +Exposes a *remote* gRPC training backend — e.g. a Google Colab run sitting behind +a raw-TCP tunnel — on a **local** port, so a local ``weightslab start`` session +can proxy to it as if it were local. + +Why a *raw* byte forwarder (no protocol parsing): the browser speaks gRPC-Web to +Envoy, and Envoy speaks native HTTP/2 gRPC to its upstream. Those HTTP/2 frames +must pass through byte-for-byte — anything that re-frames them (an HTTP/1 proxy, +a gRPC-Web tunnel) breaks the connection. So this shuttles bytes both ways and +leaves the protocol untouched. The matching remote tunnel must likewise be raw +TCP — ``bore local 50051 --to bore.pub`` (zero-signup) or ``ngrok tcp 50051`` +(needs a card on the free tier) — and the backend must run **plaintext** so no +TLS terminates mid-path. +""" + +import logging +import os +import re +import socket +import subprocess +import sys +import threading + +logger = logging.getLogger(__name__) + +# Env var read as the default ENDPOINT so a bare ``weightslab tunnel`` works +# once it is exported (e.g. the notebook can print `export WEIGHTSLAB_TUNNEL_ENDPOINT=...`). +_ENDPOINT_ENV_VAR = "WEIGHTSLAB_TUNNEL_ENDPOINT" + +# `bore` (https://github.com/ekzhang/bore) — the zero-signup raw-TCP relay used +# by the ``serving_bore`` path of ``wl.serve`` to expose a training backend +# (e.g. from Colab) to the internet so a local Weights Studio can reach it. +_BORE_VERSION = "v0.6.0" +_BORE_RELAY = "bore.pub" +# Keep spawned bore processes referenced so they are not garbage-collected +# (which would close their pipes and drop the tunnel) while the kernel lives. +_BORE_PROCS = [] + +# The default local port a local UI proxy dials (GRPC_BACKEND_PORT default). +# Binding here makes a remote backend look local to the UI server. +DEFAULT_LISTEN_PORT = 50051 + +# Size of the per-read buffer for the byte pump. +_BUF = 65536 + +# Scheme prefixes we tolerate on an endpoint string (ngrok prints ``tcp://...``). +_STRIP_SCHEMES = ("tcp://", "grpc://", "grpcs://", "http://", "https://") + + +def _default_listen_host() -> str: + """Interface to bind so the UI's Envoy container can reach the tunnel. + + Docker Desktop (Windows/macOS) routes ``host.docker.internal`` to host + loopback, so ``127.0.0.1`` is reachable *and* stays private to this machine. + On Linux the compose ``host-gateway`` resolves to the docker bridge IP, which + cannot reach a loopback-only listener — so bind all interfaces there. + """ + if sys.platform in ("win32", "darwin"): + return "127.0.0.1" + return "0.0.0.0" + + +def _parse_endpoint(endpoint: str, default_port=None): + """Parse ``[scheme://]host[:port]`` into ``(host, port)``. + + ``default_port`` is used when the endpoint has no ``:port`` (e.g. the user + passed ``--remote-port`` separately). Raises ``ValueError`` if neither is + available. + """ + ep = endpoint.strip() + for scheme in _STRIP_SCHEMES: + if ep.lower().startswith(scheme): + ep = ep[len(scheme):] + break + ep = ep.rstrip("/") + if ":" in ep: + host, _, port = ep.rpartition(":") + if not host: + raise ValueError(f"Endpoint '{endpoint}' is missing a host") + try: + return host, int(port) + except ValueError: + raise ValueError(f"Endpoint '{endpoint}' has a non-numeric port '{port}'") + if default_port is None: + raise ValueError( + f"No port in endpoint '{endpoint}' and no --remote-port given" + ) + return ep, int(default_port) + + +def _pipe(src: socket.socket, dst: socket.socket) -> None: + """Copy bytes from ``src`` to ``dst`` until either side closes.""" + try: + while True: + data = src.recv(_BUF) + if not data: + break + dst.sendall(data) + except OSError: + # Peer reset / half-open close — normal on connection teardown. + pass + finally: + # Unblock the paired pump so both sockets tear down together. + for s in (src, dst): + try: + s.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + +def _handle(client: socket.socket, remote_addr) -> None: + """Dial the remote for one accepted client and pump both directions. + + The remote is re-resolved per connection (via ``create_connection``) so a + tunnel endpoint whose DNS/IP changes is picked up without a restart. + """ + try: + remote = socket.create_connection(remote_addr, timeout=10) + except OSError as exc: + logger.warning( + f"Could not reach remote {remote_addr[0]}:{remote_addr[1]}: {exc}" + ) + try: + client.close() + except OSError: + pass + return + remote.settimeout(None) + client.settimeout(None) + threading.Thread(target=_pipe, args=(client, remote), daemon=True).start() + threading.Thread(target=_pipe, args=(remote, client), daemon=True).start() + + +def run_tunnel(remote_host: str, remote_port: int, + listen_host: str = None, + listen_port: int = DEFAULT_LISTEN_PORT) -> int: + """Forward ``listen_host:listen_port`` (local) to ``remote_host:remote_port``. + + Blocks until Ctrl+C. Returns a process exit code (0 on clean stop, 1 if the + local port could not be bound). + """ + listen_host = listen_host or _default_listen_host() + remote_addr = (remote_host, remote_port) + + # Best-effort reachability probe so the user gets immediate feedback instead + # of a silent hang when the Colab cell / ngrok tunnel isn't up yet. + try: + probe = socket.create_connection(remote_addr, timeout=8) + probe.close() + logger.info(f" Remote backend reachable at {remote_host}:{remote_port}") + except OSError as exc: + logger.warning( + f"Remote {remote_host}:{remote_port} not reachable yet ({exc}). " + "Starting anyway — is the Colab training cell running with the tunnel " + "(e.g. bore) up?" + ) + + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + srv.bind((listen_host, listen_port)) + except OSError as exc: + logger.error( + f"Cannot bind {listen_host}:{listen_port}: {exc}. " + "Another process (a local backend, or a previous tunnel) may already " + "hold it — stop it or pass a different --listen-port." + ) + srv.close() + return 1 + srv.listen(128) + + logger.info("=" * 60) + logger.info(f" Tunnel up: {listen_host}:{listen_port} -> {remote_host}:{remote_port}") + logger.info(" If the UI isn't running yet, in another terminal: weightslab start") + logger.info(" Then open the URL printed by the start command") + logger.info(" Ctrl+C to stop the tunnel.") + logger.info("=" * 60) + + try: + while True: + client, peer = srv.accept() + logger.debug(f"New connection from {peer[0]}:{peer[1]}") + _handle(client, remote_addr) + except KeyboardInterrupt: + logger.info("Tunnel stopped.") + return 0 + finally: + srv.close() + + +def tunnel_connect(args) -> None: + """`weightslab tunnel [ENDPOINT] [opts]` handler. Exits the process. + + ENDPOINT is optional: when omitted it falls back to the + ``WEIGHTSLAB_TUNNEL_ENDPOINT`` env var, so a bare ``weightslab tunnel`` + works once that is set. Everything else (local port 50051, auto listen + host) already defaults. + """ + endpoint = getattr(args, "endpoint", None) or os.environ.get(_ENDPOINT_ENV_VAR) + if not endpoint: + logger.error( + "No endpoint given. Pass it as an argument or set " + f"{_ENDPOINT_ENV_VAR}.\n" + " e.g. weightslab tunnel bore.pub:12345\n" + f" or export {_ENDPOINT_ENV_VAR}=bore.pub:12345 && weightslab tunnel" + ) + sys.exit(2) + + remote_port_arg = getattr(args, "remote_port", None) + try: + host, port = _parse_endpoint(endpoint, default_port=remote_port_arg) + except ValueError as exc: + logger.error(str(exc)) + sys.exit(2) + + listen_host = getattr(args, "listen_host", None) + listen_port = getattr(args, "listen_port", None) or DEFAULT_LISTEN_PORT + rc = run_tunnel(host, port, listen_host=listen_host, listen_port=listen_port) + sys.exit(rc or 0) + + +# --------------------------------------------------------------------------- +# Server side: expose a LOCAL backend port to the internet via a bore relay. +# This is the counterpart of the client above — it runs on the training host +# (e.g. Colab) and is what ``wl.serve(serving_bore=True)`` drives. +# --------------------------------------------------------------------------- + +def _bore_asset(version: str): + """Return ``(asset_filename, archive_kind)`` for the running platform.""" + import platform + + system = platform.system().lower() + machine = platform.machine().lower() + arch = {"x86_64": "x86_64", "amd64": "x86_64", + "aarch64": "aarch64", "arm64": "aarch64"}.get(machine) + if system == "linux" and arch: + return f"bore-{version}-{arch}-unknown-linux-musl.tar.gz", "tar" + if system == "darwin" and arch: + return f"bore-{version}-{arch}-apple-darwin.tar.gz", "tar" + if system == "windows": + return f"bore-{version}-x86_64-pc-windows-msvc.zip", "zip" + raise RuntimeError( + f"No prebuilt bore binary for {system}/{machine}. " + "Install bore manually (https://github.com/ekzhang/bore) and put it on PATH." + ) + + +def _ensure_bore(version: str = _BORE_VERSION) -> str: + """Return a path to a runnable ``bore`` binary, downloading it if needed. + + Prefers a ``bore`` already on PATH; otherwise downloads the release asset + for this platform into ``~/.weightslab/bin`` and caches it there. + """ + import shutil + from pathlib import Path + + on_path = shutil.which("bore") + if on_path: + return on_path + + cache = Path.home() / ".weightslab" / "bin" + cache.mkdir(parents=True, exist_ok=True) + exe = cache / ("bore.exe" if sys.platform == "win32" else "bore") + if exe.exists(): + return str(exe) + + import tarfile + import tempfile + import urllib.request + import zipfile + + asset, kind = _bore_asset(version) + url = f"https://github.com/ekzhang/bore/releases/download/{version}/{asset}" + logger.info(f"Downloading bore {version} ({asset})...") + tmp = Path(tempfile.mkdtemp()) + archive = tmp / asset + urllib.request.urlretrieve(url, archive) + if kind == "tar": + with tarfile.open(archive) as t: + t.extractall(tmp) + else: + with zipfile.ZipFile(archive) as z: + z.extractall(tmp) + + found = None + for p in tmp.rglob("bore*"): + if p.is_file() and p.suffix in ("", ".exe"): + found = p + break + if found is None: + raise RuntimeError(f"bore binary not found inside {asset}") + shutil.copy2(found, exe) + if sys.platform != "win32": + os.chmod(exe, 0o755) + return str(exe) + + +def serve_bore(port: int = DEFAULT_LISTEN_PORT, relay: str = _BORE_RELAY, + version: str = _BORE_VERSION, timeout: float = 30.0): + """Expose local ``port`` to the internet over a raw-TCP bore relay. + + Downloads the bore client if needed, starts ``bore local --to + `` in the background, and returns the public endpoint string (e.g. + ``"bore.pub:53984"``), or ``None`` if no endpoint was reported within + ``timeout`` seconds. Non-blocking beyond that initial handshake — the tunnel + keeps running in the background for the life of the process. + + Security note: the public relay (``bore.pub``) is shared; the random remote + port is the only thing guarding the endpoint. Fine for a demo, not for + sensitive data. + """ + try: + bore = _ensure_bore(version) + except Exception as exc: # download / platform failure — non-fatal for serve() + logger.warning(f"Could not set up bore tunnel: {exc}") + return None + + # Imported lazily: utils.tools pulls in torch, and keeping it out of this + # module's import path lets the torch-free `weightslab` CLI stay fast. + from weightslab.utils.tools import _running_in_notebook + + proc = subprocess.Popen( + [bore, "local", str(port), "--to", relay], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, start_new_session=True if _running_in_notebook() else False + ) # Bore independent of the parent process (so it keeps running if the notebook kernel restarts) + _BORE_PROCS.append(proc) + + result = {"endpoint": None} + found = threading.Event() + relay_re = re.compile(re.escape(relay) + r":(\d+)") + + def _read(): + # One reader thread both captures the endpoint and keeps draining the + # pipe afterwards, so it never fills and stalls the tunnel. + for line in proc.stdout: + if result["endpoint"] is None: + m = relay_re.search(line) + if m: + result["endpoint"] = f"{relay}:{m.group(1)}" + found.set() + + threading.Thread(target=_read, daemon=True).start() + found.wait(timeout) + if result["endpoint"] is None: + logger.warning( + f"bore did not report an endpoint within {timeout:.0f}s " + "(is the relay reachable?)." + ) + return result["endpoint"] diff --git a/weightslab/ui/__init__.py b/weightslab/ui/__init__.py new file mode 100644 index 00000000..aaaff4df --- /dev/null +++ b/weightslab/ui/__init__.py @@ -0,0 +1,10 @@ +"""Pure-Python WeightsLab UI server. + +This package bundles the pre-built Weights Studio single-page app under +``weightslab/ui/static`` and serves it together with a gRPC-Web proxy from a +single stdlib HTTP server -- no Docker, no Envoy, no nginx. + +Public entry point: :func:`weightslab.ui.server.serve_ui`. +""" + +from weightslab.ui.server import serve_ui, static_dir # noqa: F401 diff --git a/weightslab/ui/server.py b/weightslab/ui/server.py new file mode 100644 index 00000000..945162a9 --- /dev/null +++ b/weightslab/ui/server.py @@ -0,0 +1,561 @@ +"""Docker-free WeightsLab UI server. + +A single stdlib HTTP server that does everything the old Docker stack +(Envoy + nginx frontend image) used to do: + +* serves the pre-built Weights Studio SPA vendored under ``weightslab/ui/static``, +* injects a tiny runtime config so the SPA talks to *this* same origin, +* proxies gRPC-Web (both ``application/grpc-web-text`` and + ``application/grpc-web+proto``) to the running backend gRPC server -- the + exact translation Envoy performed, re-implemented generically in Python. + +The proxy is fully generic: it forwards raw protobuf bytes and never needs the +message definitions, so it keeps working when the proto changes. Every RPC is +dialed as a server-streaming call (a unary response is just a stream of one +message on the wire), which means one code path handles unary and streaming +RPCs alike. + +Only the Python stdlib and ``grpcio`` (already a hard dependency) are used -- +no new runtime dependencies. +""" + +from __future__ import annotations + +import base64 +import os +import posixpath +import socket +import ssl +import struct +import sys +import threading +import webbrowser +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Iterable, Optional, Tuple +from urllib.parse import unquote + +import grpc + +# --------------------------------------------------------------------------- # +# Constants +# --------------------------------------------------------------------------- # + +# gRPC-Web frame flags (first byte of every 5-byte frame prefix). +_FLAG_DATA = 0x00 +_FLAG_TRAILER = 0x80 + +# 256 MiB, matching the backend's grpc.max_*_message_length options. +_MAX_MESSAGE_LENGTH = 256 * 1024 * 1024 + +# Request headers we must never forward as gRPC metadata. +_HOP_BY_HOP = { + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", + "te", "trailers", "transfer-encoding", "upgrade", "host", "content-length", + "content-type", "accept", "accept-encoding", "origin", "referer", +} + + +def static_dir() -> str: + """Absolute path to the bundled SPA directory (``weightslab/ui/static``).""" + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") + + +def has_static_assets() -> bool: + """True when a real built SPA (index.html) is bundled.""" + return os.path.isfile(os.path.join(static_dir(), "index.html")) + + +# --------------------------------------------------------------------------- # +# gRPC-Web framing helpers +# --------------------------------------------------------------------------- # + +def _iter_frames(payload: bytes) -> Iterable[Tuple[int, bytes]]: + """Yield ``(flag, message)`` tuples from a gRPC-Web framed byte string.""" + offset = 0 + n = len(payload) + while offset + 5 <= n: + flag = payload[offset] + (length,) = struct.unpack(">I", payload[offset + 1:offset + 5]) + offset += 5 + message = payload[offset:offset + length] + offset += length + yield flag, message + + +def _first_message(payload: bytes) -> bytes: + """Extract the first data message from a gRPC-Web request body.""" + for flag, message in _iter_frames(payload): + if not (flag & _FLAG_TRAILER): + return message + return b"" + + +def _data_frame(message: bytes) -> bytes: + return bytes([_FLAG_DATA]) + struct.pack(">I", len(message)) + message + + +def _trailer_frame(status: int, message: str) -> bytes: + # gRPC-Web trailers are HTTP/1.1-style header lines in the frame payload. + safe = message.replace("\r", " ").replace("\n", " ") + text = f"grpc-status:{status}\r\ngrpc-message:{safe}\r\n".encode("utf-8") + return bytes([_FLAG_TRAILER]) + struct.pack(">I", len(text)) + text + + +# --------------------------------------------------------------------------- # +# Request handler +# --------------------------------------------------------------------------- # + +class _UIRequestHandler(BaseHTTPRequestHandler): + """Serves the SPA and proxies gRPC-Web to the backend gRPC server.""" + + protocol_version = "HTTP/1.1" + server_version = "WeightsLabUI" + + # Injected by the factory in :func:`serve_ui`. + api_prefix: str = "/api" + static_root: str = "" + channel: "grpc.Channel" = None # type: ignore[assignment] + grpc_auth_token: Optional[str] = None + rpc_timeout: float = 300.0 + + # -- logging: quiet by default, honour WEIGHTSLAB_UI_VERBOSE ------------- # + def log_message(self, fmt, *args): # noqa: D401 + if os.getenv("WEIGHTSLAB_UI_VERBOSE"): + sys.stderr.write("[weightslab-ui] %s - %s\n" + % (self.address_string(), fmt % args)) + + # -- CORS (harmless same-origin; also enables `vite dev` against us) ----- # + def _send_cors(self): + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") + self.send_header( + "Access-Control-Allow-Headers", + "content-type,x-grpc-web,x-user-agent,grpc-timeout,authorization," + "x-grpc-token", + ) + self.send_header( + "Access-Control-Expose-Headers", + "grpc-status,grpc-message,grpc-status-details-bin", + ) + + def do_OPTIONS(self): # noqa: N802 + self.send_response(HTTPStatus.NO_CONTENT) + self._send_cors() + self.send_header("Content-Length", "0") + self.end_headers() + + def do_GET(self): # noqa: N802 + self._serve_static() + + def do_HEAD(self): # noqa: N802 + self._serve_static(head_only=True) + + def do_POST(self): # noqa: N802 + path = self.path.split("?", 1)[0] + if path.startswith(self.api_prefix + "/") or path == self.api_prefix: + self._proxy_grpc_web(path) + else: + self._send_simple(HTTPStatus.NOT_FOUND, "Not found") + + # ------------------------------------------------------------------ # + # gRPC-Web proxy + # ------------------------------------------------------------------ # + def _proxy_grpc_web(self, path: str): + # Strip the API prefix to recover the gRPC method path, e.g. + # /api/ExperimentService/GetWeights -> /ExperimentService/GetWeights + method_path = path[len(self.api_prefix):] + if not method_path.startswith("/"): + method_path = "/" + method_path + + content_type = (self.headers.get("Content-Type") or "").lower() + is_text = "grpc-web-text" in content_type + + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + length = 0 + raw = self.rfile.read(length) if length else b"" + body = base64.b64decode(raw) if (is_text and raw) else raw + request_message = _first_message(body) + + metadata = self._collect_metadata() + + stub = self.channel.unary_stream( + method_path, + request_serializer=lambda b: b, + response_deserializer=lambda b: b, + ) + + out = bytearray() + status_code = 0 + status_message = "" + try: + call = stub(request_message, metadata=metadata, + timeout=self.rpc_timeout) + for message in call: + out += _data_frame(message) + except grpc.RpcError as err: + status_code = int(err.code().value[0]) if err.code() else 2 + status_message = err.details() or str(err) + except Exception as err: # pragma: no cover - defensive + status_code = 2 # UNKNOWN + status_message = str(err) + + out += _trailer_frame(status_code, status_message) + payload = base64.b64encode(bytes(out)) if is_text else bytes(out) + + resp_content_type = ( + "application/grpc-web-text" if is_text + else "application/grpc-web+proto" + ) + self.send_response(HTTPStatus.OK) + self._send_cors() + self.send_header("Content-Type", resp_content_type) + self.send_header("Content-Length", str(len(payload))) + # Mirror status in HTTP headers too (belt and suspenders). + self.send_header("grpc-status", str(status_code)) + if status_message: + self.send_header("grpc-message", status_message) + self.end_headers() + if payload: + self.wfile.write(payload) + + def _collect_metadata(self): + metadata = [] + for key, value in self.headers.items(): + lkey = key.lower() + if lkey in _HOP_BY_HOP or lkey.startswith("access-control"): + continue + # grpc metadata keys must be ascii lowercase; skip pseudo headers. + if lkey.startswith(":"): + continue + metadata.append((lkey, value)) + if self.grpc_auth_token: + metadata.append(("x-grpc-token", self.grpc_auth_token)) + metadata.append(("authorization", f"Bearer {self.grpc_auth_token}")) + return metadata + + # ------------------------------------------------------------------ # + # Static SPA serving (with SPA fallback to index.html) + # ------------------------------------------------------------------ # + def _resolve_static_path(self, url_path: str) -> Optional[str]: + url_path = unquote(url_path.split("?", 1)[0].split("#", 1)[0]) + # Normalise and prevent path traversal. + norm = posixpath.normpath(url_path) + parts = [p for p in norm.split("/") if p not in ("", ".", "..")] + candidate = os.path.join(self.static_root, *parts) + if os.path.isdir(candidate): + candidate = os.path.join(candidate, "index.html") + return candidate + + def _serve_static(self, head_only: bool = False): + if not self.static_root or not os.path.isdir(self.static_root): + self._send_simple( + HTTPStatus.SERVICE_UNAVAILABLE, + "Weights Studio UI assets are not bundled in this install.", + ) + return + + url_path = self.path.split("?", 1)[0] + candidate = self._resolve_static_path(url_path) + index_path = os.path.join(self.static_root, "index.html") + + if candidate and os.path.isfile(candidate): + target = candidate + else: + # SPA fallback: any unknown route serves index.html. + target = index_path + + if not os.path.isfile(target): + self._send_simple(HTTPStatus.NOT_FOUND, "Not found") + return + + is_index = os.path.abspath(target) == os.path.abspath(index_path) + if is_index: + data = self._render_index(target) + ctype = "text/html; charset=utf-8" + else: + with open(target, "rb") as fh: + data = fh.read() + ctype = _guess_type(target) + + self.send_response(HTTPStatus.OK) + self._send_cors() + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(data))) + if is_index: + self.send_header("Cache-Control", "no-cache") + self.end_headers() + if not head_only: + self.wfile.write(data) + + def _render_index(self, index_path: str) -> bytes: + with open(index_path, "r", encoding="utf-8") as fh: + html = fh.read() + token_js = ( + f"window.GRPC_AUTH_TOKEN={_js_str(self.grpc_auth_token)};" + "window.WS_ENABLE_GRPC_AUTH_TOKEN='1';" + if self.grpc_auth_token + else "window.WS_ENABLE_GRPC_AUTH_TOKEN='0';" + ) + # Self-configuring: point the SPA at *this* origin's /api, whatever the + # host/port/scheme happens to be (no rebuild). Plus the feature/cache + # runtime globals from env vars — the faithful replacement for the old + # nginx config.js, so the UI stays tunable without a rebuild or Docker. + config = ( + "" + ) + if "" in html: + html = html.replace("", config + "", 1) + else: + html = config + html + return html.encode("utf-8") + + def _send_simple(self, status: HTTPStatus, message: str): + data = message.encode("utf-8") + self.send_response(status) + self._send_cors() + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + +def _js_str(value: Optional[str]) -> str: + if value is None: + return "''" + escaped = value.replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + + +# Runtime UI-config globals mirrored from environment variables — the faithful, +# Docker-free replacement for the old nginx entrypoint's config.js. Each +# ``window.WS_*`` global is set from the FIRST environment variable present in +# its candidate list, so a deployer can tune the UI at launch time without +# rebuilding. When none are set the global is omitted and the SPA falls back to +# its own built-in default. +_UI_ENV_GLOBALS = [ + ("WS_HISTOGRAM_MAX_BINS", ("WS_HISTOGRAM_MAX_BINS", "VITE_HISTOGRAM_MAX_BINS")), + ("WS_BB_THUMB_RENDER", ("BB_THUMB_RENDER", "WS_BB_THUMB_RENDER", "VITE_BB_THUMB_RENDER")), + ("WS_BB_MODAL_RENDER", ("BB_MODAL_RENDER", "WS_BB_MODAL_RENDER", "VITE_BB_MODAL_RENDER")), + ("WS_GRID_WINDOW_SIZE", ("GRID_WINDOW_SIZE", "VITE_GRID_WINDOW_SIZE")), + ("WS_MAX_IMAGE_CACHE_SIZE", ("GRID_MAX_IMAGE_CACHE_SIZE", "VITE_WS_MAX_IMAGE_CACHE_SIZE")), + ("WS_GRID_CACHE_MAX_MB", ("GRID_CACHE_MAX_MB", "VITE_WS_GRID_CACHE_MAX_MB")), + ("WS_MODAL_CACHE_MAX_MB", ("MODAL_CACHE_MAX_MB", "VITE_WS_MODAL_CACHE_MAX_MB")), + ("WS_WL_PC_MAX_POINTS", ("PC_MAX_POINTS", "VITE_WL_PC_MAX_POINTS")), + ("WS_WL_DISABLE_GPU_RENDERING", ("DISABLE_GPU_RENDERING", "VITE_WL_DISABLE_GPU_RENDERING")), + ("WS_ENABLE_PLOTS", ("ENABLE_PLOTS", "WS_ENABLE_PLOTS", "VITE_ENABLE_PLOTS")), + ("WS_ENABLE_DATA_EXPLORATION", + ("ENABLE_DATA_EXPLORATION", "WS_ENABLE_DATA_EXPLORATION", "VITE_ENABLE_DATA_EXPLORATION")), + ("WS_ENABLE_HYPERPARAMETERS_OPTIMIZATION", + ("ENABLE_HYPERPARAMETERS_OPTIMIZATION", "WS_ENABLE_HYPERPARAMETERS_OPTIMIZATION", + "VITE_ENABLE_HYPERPARAMETERS_OPTIMIZATION")), + ("WS_ENABLE_AGENT", ("ENABLE_AGENT", "WS_ENABLE_AGENT", "VITE_ENABLE_AGENT")), +] + + +def _ui_env_globals_js() -> str: + """Build ``window.WS_*=...;`` assignments for any configured env vars (else empty).""" + parts = [] + for window_key, candidates in _UI_ENV_GLOBALS: + for env_name in candidates: + val = os.environ.get(env_name) + if val: + parts.append(f"window.{window_key}={_js_str(val)};") + break + return "".join(parts) + + +def _guess_type(path: str) -> str: + import mimetypes + ctype, _ = mimetypes.guess_type(path) + if ctype is None: + # Common web types mimetypes sometimes misses. + ext = os.path.splitext(path)[1].lower() + ctype = { + ".js": "text/javascript", + ".mjs": "text/javascript", + ".wasm": "application/wasm", + ".map": "application/json", + }.get(ext, "application/octet-stream") + if ctype.startswith("text/") and "charset" not in ctype: + ctype += "; charset=utf-8" + return ctype + + +# --------------------------------------------------------------------------- # +# gRPC upstream channel +# --------------------------------------------------------------------------- # + +def _build_channel(backend_host: str, backend_port: int, + certs_dir: Optional[str]) -> "grpc.Channel": + target = f"{backend_host}:{backend_port}" + options = [ + ("grpc.max_send_message_length", _MAX_MESSAGE_LENGTH), + ("grpc.max_receive_message_length", _MAX_MESSAGE_LENGTH), + ] + if certs_dir: + ca = os.path.join(certs_dir, "ca.crt") + client_crt = os.path.join(certs_dir, "ui-client.crt") + client_key = os.path.join(certs_dir, "ui-client.key") + if os.path.isfile(ca): + root = _read(ca) + key = _read(client_key) if os.path.isfile(client_key) else None + crt = _read(client_crt) if os.path.isfile(client_crt) else None + creds = grpc.ssl_channel_credentials( + root_certificates=root, + private_key=key, + certificate_chain=crt, + ) + return grpc.secure_channel(target, creds, options=options) + return grpc.insecure_channel(target, options=options) + + +def _read(path: str) -> bytes: + with open(path, "rb") as fh: + return fh.read() + + +# --------------------------------------------------------------------------- # +# Public entry point +# --------------------------------------------------------------------------- # + +def serve_ui( + ui_host: str = "0.0.0.0", + ui_port: int = 8080, + backend_host: str = "localhost", + backend_port: int = 50051, + open_browser: bool = True, + certs_dir: Optional[str] = None, + grpc_auth_token: Optional[str] = None, + block: bool = True, +) -> ThreadingHTTPServer: + """Start the Docker-free WeightsLab UI server. + + Serves the bundled SPA and proxies gRPC-Web to ``backend_host:backend_port``. + + Parameters + ---------- + ui_host, ui_port: + Interface / port the HTTP server binds. Open ``http://:``. + backend_host, backend_port: + The running WeightsLab gRPC backend (``wl.serve``) to proxy to. + open_browser: + Open the default web browser at the UI URL once the server is up. + certs_dir: + Optional ``WEIGHTSLAB_CERTS_DIR``. When it contains TLS material the + server serves HTTPS downstream and dials the backend with mTLS. + grpc_auth_token: + Optional gRPC auth token forwarded to the backend and exposed to the SPA. + block: + When True (default) serve forever; otherwise return immediately with the + server running in a daemon thread. + """ + root = static_dir() + if not has_static_assets(): + sys.stderr.write( + "\n[weightslab] WARNING: no built UI assets found at\n" + f" {root}\n" + " The server will still proxy gRPC-Web, but the web page will be\n" + " unavailable. Build the frontend and vendor it into this package\n" + " (see weights_studio: `npm run build:embed`).\n\n" + ) + + channel = _build_channel(backend_host, backend_port, certs_dir) + + # Determine downstream TLS from cert presence. + ssl_ctx = None + scheme = "http" + if certs_dir: + server_crt = os.path.join(certs_dir, "ui-server.crt") + server_key = os.path.join(certs_dir, "ui-server.key") + if os.path.isfile(server_crt) and os.path.isfile(server_key): + ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_ctx.load_cert_chain(server_crt, server_key) + scheme = "https" + + handler_cls = type( + "_BoundUIRequestHandler", + (_UIRequestHandler,), + { + "static_root": root, + "channel": channel, + "api_prefix": "/api", + "grpc_auth_token": grpc_auth_token, + }, + ) + + httpd = ThreadingHTTPServer((ui_host, ui_port), handler_cls) + httpd.daemon_threads = True + if ssl_ctx is not None: + httpd.socket = ssl_ctx.wrap_socket(httpd.socket, server_side=True) + + # Use the port actually bound (handles ui_port=0 and any late fallback). + ui_port = httpd.server_address[1] + display_host = "localhost" if ui_host in ("0.0.0.0", "::", "") else ui_host + url = f"{scheme}://{display_host}:{ui_port}" + + sys.stdout.write( + "\n" + " WeightsLab UI is running:\n" + f" {url}\n" + f" proxying gRPC-Web -> {backend_host}:{backend_port}\n" + " Press Ctrl+C to stop.\n\n" + ) + sys.stdout.flush() + + if open_browser: + threading.Timer(0.6, lambda: _safe_open(url)).start() + + if not block: + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + return httpd + + try: + httpd.serve_forever() + except KeyboardInterrupt: + sys.stdout.write("\n Shutting down WeightsLab UI...\n") + finally: + httpd.shutdown() + httpd.server_close() + return httpd + + +def _safe_open(url: str): + try: + webbrowser.open(url) + except Exception: + pass + + +def find_free_port(preferred: int, host: str = "0.0.0.0") -> int: + """Return ``preferred`` if bindable, else an OS-assigned free port.""" + if preferred <= 0: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.bind((host, 0)) + return sock.getsockname()[1] + finally: + sock.close() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((host, preferred)) + return preferred + except OSError: + sock.close() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind((host, 0)) + port = sock.getsockname()[1] + return port + finally: + sock.close() diff --git a/weightslab/docker/docker/utils/generate-certs-auth-token.ps1 b/weightslab/ui/utils/generate-certs-auth-token.ps1 similarity index 66% rename from weightslab/docker/docker/utils/generate-certs-auth-token.ps1 rename to weightslab/ui/utils/generate-certs-auth-token.ps1 index caa7bded..dea6737f 100644 --- a/weightslab/docker/docker/utils/generate-certs-auth-token.ps1 +++ b/weightslab/ui/utils/generate-certs-auth-token.ps1 @@ -24,7 +24,7 @@ New-Item -ItemType Directory -Force -Path $userCertDir | Out-Null # Check if certs already exist in user directory $certsExist = (Test-Path (Join-Path $userCertDir "ca.crt")) -and ` - (Test-Path (Join-Path $userCertDir "envoy-server.crt")) -and ` + (Test-Path (Join-Path $userCertDir "ui-server.crt")) -and ` (Test-Path (Join-Path $userCertDir "backend-server.crt")) # If certs exist and not forcing recreation, skip generation @@ -50,40 +50,40 @@ try { @" subjectAltName = DNS:localhost,IP:127.0.0.1,IP:0:0:0:0:0:0:0:1 extendedKeyUsage = serverAuth -"@ | Set-Content -Path (Join-Path $tmpDir "envoy-server.ext") -Encoding ASCII +"@ | Set-Content -Path (Join-Path $tmpDir "ui-server.ext") -Encoding ASCII @" - subjectAltName = DNS:localhost,DNS:host.docker.internal,IP:127.0.0.1,IP:0:0:0:0:0:0:0:1 + subjectAltName = DNS:localhost,IP:127.0.0.1,IP:0:0:0:0:0:0:0:1 extendedKeyUsage = serverAuth "@ | Set-Content -Path (Join-Path $tmpDir "backend-server.ext") -Encoding ASCII @" extendedKeyUsage = clientAuth -"@ | Set-Content -Path (Join-Path $tmpDir "envoy-client.ext") -Encoding ASCII +"@ | Set-Content -Path (Join-Path $tmpDir "ui-client.ext") -Encoding ASCII - Write-Host "Generating Envoy HTTPS server cert..." - & openssl genrsa -out (Join-Path $tmpDir "envoy-server.key") 2048 - & openssl req -new -key (Join-Path $tmpDir "envoy-server.key") -subj "/CN=localhost" -out (Join-Path $tmpDir "envoy-server.csr") - & openssl x509 -req -in (Join-Path $tmpDir "envoy-server.csr") -CA (Join-Path $tmpDir "ca.crt") -CAkey (Join-Path $tmpDir "ca.key") ` - -CAcreateserial -out (Join-Path $tmpDir "envoy-server.crt") -days 825 -sha256 -extfile (Join-Path $tmpDir "envoy-server.ext") + Write-Host "Generating UI HTTPS server cert..." + & openssl genrsa -out (Join-Path $tmpDir "ui-server.key") 2048 + & openssl req -new -key (Join-Path $tmpDir "ui-server.key") -subj "/CN=localhost" -out (Join-Path $tmpDir "ui-server.csr") + & openssl x509 -req -in (Join-Path $tmpDir "ui-server.csr") -CA (Join-Path $tmpDir "ca.crt") -CAkey (Join-Path $tmpDir "ca.key") ` + -CAcreateserial -out (Join-Path $tmpDir "ui-server.crt") -days 825 -sha256 -extfile (Join-Path $tmpDir "ui-server.ext") Write-Host "Generating backend gRPC server cert..." & openssl genrsa -out (Join-Path $tmpDir "backend-server.key") 2048 - & openssl req -new -key (Join-Path $tmpDir "backend-server.key") -subj "/CN=host.docker.internal" -out (Join-Path $tmpDir "backend-server.csr") + & openssl req -new -key (Join-Path $tmpDir "backend-server.key") -subj "/CN=localhost" -out (Join-Path $tmpDir "backend-server.csr") & openssl x509 -req -in (Join-Path $tmpDir "backend-server.csr") -CA (Join-Path $tmpDir "ca.crt") -CAkey (Join-Path $tmpDir "ca.key") ` -CAcreateserial -out (Join-Path $tmpDir "backend-server.crt") -days 825 -sha256 -extfile (Join-Path $tmpDir "backend-server.ext") - Write-Host "Generating Envoy mTLS client cert..." - & openssl genrsa -out (Join-Path $tmpDir "envoy-client.key") 2048 - & openssl req -new -key (Join-Path $tmpDir "envoy-client.key") -subj "/CN=envoy-client" -out (Join-Path $tmpDir "envoy-client.csr") - & openssl x509 -req -in (Join-Path $tmpDir "envoy-client.csr") -CA (Join-Path $tmpDir "ca.crt") -CAkey (Join-Path $tmpDir "ca.key") ` - -CAcreateserial -out (Join-Path $tmpDir "envoy-client.crt") -days 825 -sha256 -extfile (Join-Path $tmpDir "envoy-client.ext") + Write-Host "Generating UI mTLS client cert..." + & openssl genrsa -out (Join-Path $tmpDir "ui-client.key") 2048 + & openssl req -new -key (Join-Path $tmpDir "ui-client.key") -subj "/CN=ui-client" -out (Join-Path $tmpDir "ui-client.csr") + & openssl x509 -req -in (Join-Path $tmpDir "ui-client.csr") -CA (Join-Path $tmpDir "ca.crt") -CAkey (Join-Path $tmpDir "ca.key") ` + -CAcreateserial -out (Join-Path $tmpDir "ui-client.crt") -days 825 -sha256 -extfile (Join-Path $tmpDir "ui-client.ext") Copy-Item -Force (Join-Path $tmpDir "ca.crt") (Join-Path $userCertDir "ca.crt") - Copy-Item -Force (Join-Path $tmpDir "envoy-server.crt") (Join-Path $userCertDir "envoy-server.crt") - Copy-Item -Force (Join-Path $tmpDir "envoy-server.key") (Join-Path $userCertDir "envoy-server.key") - Copy-Item -Force (Join-Path $tmpDir "envoy-client.crt") (Join-Path $userCertDir "envoy-client.crt") - Copy-Item -Force (Join-Path $tmpDir "envoy-client.key") (Join-Path $userCertDir "envoy-client.key") + Copy-Item -Force (Join-Path $tmpDir "ui-server.crt") (Join-Path $userCertDir "ui-server.crt") + Copy-Item -Force (Join-Path $tmpDir "ui-server.key") (Join-Path $userCertDir "ui-server.key") + Copy-Item -Force (Join-Path $tmpDir "ui-client.crt") (Join-Path $userCertDir "ui-client.crt") + Copy-Item -Force (Join-Path $tmpDir "ui-client.key") (Join-Path $userCertDir "ui-client.key") Copy-Item -Force (Join-Path $tmpDir "backend-server.crt") (Join-Path $userCertDir "backend-server.crt") Copy-Item -Force (Join-Path $tmpDir "backend-server.key") (Join-Path $userCertDir "backend-server.key") diff --git a/weightslab/docker/docker/utils/generate-certs-auth-token.sh b/weightslab/ui/utils/generate-certs-auth-token.sh similarity index 68% rename from weightslab/docker/docker/utils/generate-certs-auth-token.sh rename to weightslab/ui/utils/generate-certs-auth-token.sh index c45a8d68..6c8f2f2a 100644 --- a/weightslab/docker/docker/utils/generate-certs-auth-token.sh +++ b/weightslab/ui/utils/generate-certs-auth-token.sh @@ -44,7 +44,7 @@ mkdir -p "$USER_CERT_DIR" CERTS_EXIST=false if [ -f "$USER_CERT_DIR/ca.crt" ] && \ - [ -f "$USER_CERT_DIR/envoy-server.crt" ] && \ + [ -f "$USER_CERT_DIR/ui-server.crt" ] && \ [ -f "$USER_CERT_DIR/backend-server.crt" ]; then CERTS_EXIST=true fi @@ -74,43 +74,43 @@ openssl req -x509 -new -nodes -key "$TMP_DIR/ca.key" -sha256 -days 825 \ -subj "/CN=weightslab-dev-ca" \ -out "$TMP_DIR/ca.crt" -cat > "$TMP_DIR/envoy-server.ext" << 'EOF' +cat > "$TMP_DIR/ui-server.ext" << 'EOF' subjectAltName = DNS:localhost,IP:127.0.0.1,IP:0:0:0:0:0:0:0:1 extendedKeyUsage = serverAuth EOF cat > "$TMP_DIR/backend-server.ext" << 'EOF' -subjectAltName = DNS:localhost,DNS:host.docker.internal,IP:127.0.0.1,IP:0:0:0:0:0:0:0:1 +subjectAltName = DNS:localhost,IP:127.0.0.1,IP:0:0:0:0:0:0:0:1 extendedKeyUsage = serverAuth EOF -cat > "$TMP_DIR/envoy-client.ext" << 'EOF' +cat > "$TMP_DIR/ui-client.ext" << 'EOF' extendedKeyUsage = clientAuth EOF -echo "Generating Envoy HTTPS server cert..." -openssl genrsa -out "$TMP_DIR/envoy-server.key" 2048 -openssl req -new -key "$TMP_DIR/envoy-server.key" -subj "/CN=localhost" -out "$TMP_DIR/envoy-server.csr" -openssl x509 -req -in "$TMP_DIR/envoy-server.csr" -CA "$TMP_DIR/ca.crt" -CAkey "$TMP_DIR/ca.key" \ - -CAcreateserial -out "$TMP_DIR/envoy-server.crt" -days 825 -sha256 -extfile "$TMP_DIR/envoy-server.ext" +echo "Generating UI HTTPS server cert..." +openssl genrsa -out "$TMP_DIR/ui-server.key" 2048 +openssl req -new -key "$TMP_DIR/ui-server.key" -subj "/CN=localhost" -out "$TMP_DIR/ui-server.csr" +openssl x509 -req -in "$TMP_DIR/ui-server.csr" -CA "$TMP_DIR/ca.crt" -CAkey "$TMP_DIR/ca.key" \ + -CAcreateserial -out "$TMP_DIR/ui-server.crt" -days 825 -sha256 -extfile "$TMP_DIR/ui-server.ext" echo "Generating backend gRPC server cert..." openssl genrsa -out "$TMP_DIR/backend-server.key" 2048 -openssl req -new -key "$TMP_DIR/backend-server.key" -subj "/CN=host.docker.internal" -out "$TMP_DIR/backend-server.csr" +openssl req -new -key "$TMP_DIR/backend-server.key" -subj "/CN=localhost" -out "$TMP_DIR/backend-server.csr" openssl x509 -req -in "$TMP_DIR/backend-server.csr" -CA "$TMP_DIR/ca.crt" -CAkey "$TMP_DIR/ca.key" \ -CAcreateserial -out "$TMP_DIR/backend-server.crt" -days 825 -sha256 -extfile "$TMP_DIR/backend-server.ext" -echo "Generating Envoy mTLS client cert..." -openssl genrsa -out "$TMP_DIR/envoy-client.key" 2048 -openssl req -new -key "$TMP_DIR/envoy-client.key" -subj "/CN=envoy-client" -out "$TMP_DIR/envoy-client.csr" -openssl x509 -req -in "$TMP_DIR/envoy-client.csr" -CA "$TMP_DIR/ca.crt" -CAkey "$TMP_DIR/ca.key" \ - -CAcreateserial -out "$TMP_DIR/envoy-client.crt" -days 825 -sha256 -extfile "$TMP_DIR/envoy-client.ext" +echo "Generating UI mTLS client cert..." +openssl genrsa -out "$TMP_DIR/ui-client.key" 2048 +openssl req -new -key "$TMP_DIR/ui-client.key" -subj "/CN=ui-client" -out "$TMP_DIR/ui-client.csr" +openssl x509 -req -in "$TMP_DIR/ui-client.csr" -CA "$TMP_DIR/ca.crt" -CAkey "$TMP_DIR/ca.key" \ + -CAcreateserial -out "$TMP_DIR/ui-client.crt" -days 825 -sha256 -extfile "$TMP_DIR/ui-client.ext" cp -f "$TMP_DIR/ca.crt" "$USER_CERT_DIR/ca.crt" -cp -f "$TMP_DIR/envoy-server.crt" "$USER_CERT_DIR/envoy-server.crt" -cp -f "$TMP_DIR/envoy-server.key" "$USER_CERT_DIR/envoy-server.key" -cp -f "$TMP_DIR/envoy-client.crt" "$USER_CERT_DIR/envoy-client.crt" -cp -f "$TMP_DIR/envoy-client.key" "$USER_CERT_DIR/envoy-client.key" +cp -f "$TMP_DIR/ui-server.crt" "$USER_CERT_DIR/ui-server.crt" +cp -f "$TMP_DIR/ui-server.key" "$USER_CERT_DIR/ui-server.key" +cp -f "$TMP_DIR/ui-client.crt" "$USER_CERT_DIR/ui-client.crt" +cp -f "$TMP_DIR/ui-client.key" "$USER_CERT_DIR/ui-client.key" cp -f "$TMP_DIR/backend-server.crt" "$USER_CERT_DIR/backend-server.crt" cp -f "$TMP_DIR/backend-server.key" "$USER_CERT_DIR/backend-server.key" diff --git a/weightslab/ui/utils/sync-frontend.sh b/weightslab/ui/utils/sync-frontend.sh new file mode 100644 index 00000000..6db41053 --- /dev/null +++ b/weightslab/ui/utils/sync-frontend.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# Vendor the built Weights Studio SPA into this package so `weightslab start` +# can serve it (and so it ships inside the wheel). Docker-free. +# +# The frontend build lives in the weights_studio repo (its ./dist, gitignored). +# This script builds it there (unless SKIP_BUILD=1) and copies dist/ into +# weightslab/ui/static/ here. +# +# Usage: +# weightslab/ui/utils/sync-frontend.sh +# WEIGHTS_STUDIO_DIR=../weights_studio SKIP_BUILD=1 weightslab/ui/utils/sync-frontend.sh +# +# Env: +# WEIGHTS_STUDIO_DIR Path to the weights_studio repo (default: ../weights_studio +# relative to the weightslab repo root). +# SKIP_BUILD=1 Reuse the existing dist/ instead of rebuilding. + +set -e + +# weightslab repo root = three levels up from this script (weightslab/ui/utils/). +PKG_STATIC_DIR="$(cd "$(dirname "$0")/.." && pwd)/static" +WL_REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +STUDIO_DIR="${WEIGHTS_STUDIO_DIR:-$WL_REPO_ROOT/../weights_studio}" + +if [ ! -d "$STUDIO_DIR" ]; then + echo "ERROR: weights_studio repo not found at: $STUDIO_DIR" >&2 + echo " Set WEIGHTS_STUDIO_DIR to the weights_studio repo root." >&2 + exit 1 +fi +STUDIO_DIR="$(cd "$STUDIO_DIR" && pwd)" + +if [ "${SKIP_BUILD:-0}" = "1" ]; then + echo "[sync-frontend] SKIP_BUILD=1 -> reusing existing dist/" +else + echo "[sync-frontend] building SPA in $STUDIO_DIR ..." + ( cd "$STUDIO_DIR" && npm run build ) +fi + +if [ ! -d "$STUDIO_DIR/dist" ]; then + echo "ERROR: no dist/ at $STUDIO_DIR/dist (build first)" >&2 + exit 1 +fi + +echo "[sync-frontend] vendoring dist/ -> $PKG_STATIC_DIR" +rm -rf "$PKG_STATIC_DIR" +mkdir -p "$PKG_STATIC_DIR" +cp -R "$STUDIO_DIR/dist/." "$PKG_STATIC_DIR/" + +echo "[sync-frontend] done. 'weightslab start' will now serve the updated UI." diff --git a/weightslab/ui_docker_bridge.py b/weightslab/ui_docker_bridge.py deleted file mode 100644 index 5fa795e0..00000000 --- a/weightslab/ui_docker_bridge.py +++ /dev/null @@ -1,1225 +0,0 @@ -"""Weights Studio Docker management with secure TLS.""" - -import argparse -import os -import re -import shutil -import stat -import subprocess -import sys -import socket -import logging -from pathlib import Path - -from weightslab.security import CertAuthManager - -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -logger = logging.getLogger(__name__) - -# Capture whether WEIGHTSLAB_CERTS_DIR was already in the shell env when this -# process started (set by the user) vs. injected later by our own code. -_CERTS_DIR_IN_ORIGINAL_ENV: bool = "WEIGHTSLAB_CERTS_DIR" in os.environ - -# Docker resources owned by the bundled stack. Cleanup is scoped strictly to -# these names — we never run a global `docker system prune`. -_FRONTEND_IMAGE = "graybx/weightslab" -_STACK_CONTAINERS = ("weights_studio_envoy", "weights_studio_frontend") - -# Default frontend image repo (no tag) and tag. Overridable per-launch via the -# `--image/-i` and `--version/-v` flags on `weightslab ui launch`, which are -# passed to docker compose through the WS_FRONTEND_IMAGE env var (see the -# `${WS_FRONTEND_IMAGE:-...}` substitution in docker-compose.yml). -_DEFAULT_FRONTEND_REPO = "graybx/weightslab" -_DEFAULT_FRONTEND_TAG = "latest" - - -def _resolve_frontend_image(image_arg=None, version_arg=None) -> str: - """Return the full ``repo:tag`` frontend image ref from the CLI flags. - - ``image_arg`` is the repo (e.g. ``guillaumep2705/weightslab``) and may itself - carry a tag; ``version_arg`` is the tag (e.g. ``latest`` or ``v1.2.3``). An - explicit ``--version`` always wins over any tag embedded in ``--image``. - Falls back to ``graybx/weightslab:latest`` when neither is given. - """ - image = image_arg or _DEFAULT_FRONTEND_REPO - # A ':' in the last path segment is a tag (not a registry port like host:5000/img). - last_segment = image.rsplit("/", 1)[-1] - has_tag = ":" in last_segment - if version_arg: - repo = image.rsplit(":", 1)[0] if has_tag else image - return f"{repo}:{version_arg}" - if has_tag: - return image - return f"{image}:{_DEFAULT_FRONTEND_TAG}" - -# Cached Docker Compose base command. Resolved once per process to either the -# v2 plugin (``["docker", "compose"]``) or the legacy v1 standalone binary -# (``["docker-compose"]``) — see _detect_compose_cmd(). None until first probe. -_COMPOSE_BASE_CMD = None - -# TLS/auth env vars that are *derived* from cert-file presence in -# WEIGHTSLAB_CERTS_DIR. WEIGHTSLAB_CERTS_DIR is the single source of truth; these -# are computed by the deploy pipeline (build-and-deploy.sh + the compose `auto` -# logic) from the actual files. They are stripped before launching so a stale or -# pre-set value can never override the file-based decision. -_DERIVED_DEPLOY_ENV_VARS = ( - "GRPC_TLS_ENABLED", - "GRPC_TLS_REQUIRE_CLIENT_AUTH", - "GRPC_TLS_CERT_FILE", - "GRPC_TLS_KEY_FILE", - "GRPC_TLS_CA_FILE", - "ENVOY_UPSTREAM_TLS", - "ENVOY_DOWNSTREAM_TLS", - "WS_SERVER_PROTOCOL", - "VITE_SERVER_PROTOCOL", - "VITE_DEV_SERVER_HTTPS", - "WL_ENABLE_GRPC_AUTH_TOKEN", - "VITE_WL_ENABLE_GRPC_AUTH_TOKEN", - "GRPC_AUTH_TOKEN", - "VITE_GRPC_AUTH_TOKEN", -) - - -def _persist_certs_dir(certs_dir_str: str) -> None: - """Persist WEIGHTSLAB_CERTS_DIR so future terminals and the training backend find it. - - Windows — runs `setx` (permanent user env) and prints the PS one-liner for - the current session. - Linux/macOS — appends an export line to ~/.bashrc (idempotent) and prints the - source command for the current session. - """ - export_line = f'export WEIGHTSLAB_CERTS_DIR="{certs_dir_str}"' - if _is_windows(): - result = subprocess.run( - ["setx", "WEIGHTSLAB_CERTS_DIR", certs_dir_str], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - ) - if result.returncode == 0: - logger.info(" WEIGHTSLAB_CERTS_DIR saved permanently via setx (new terminals will have it)") - else: - logger.warning(f"setx failed — set it manually: setx WEIGHTSLAB_CERTS_DIR \"{certs_dir_str}\"") - logger.info(f" Current terminal (PowerShell): $env:WEIGHTSLAB_CERTS_DIR = \"{certs_dir_str}\"") - else: - bashrc = Path.home() / ".bashrc" - try: - existing = bashrc.read_text(encoding="utf-8") if bashrc.exists() else "" - if export_line not in existing: - with open(bashrc, "a", encoding="utf-8") as f: - f.write(f"\n# Added by weightslab\n{export_line}\n") - logger.info(f" WEIGHTSLAB_CERTS_DIR appended to {bashrc} (new terminals will have it)") - else: - logger.info(f" WEIGHTSLAB_CERTS_DIR already in {bashrc}") - except OSError as e: - logger.warning(f"Could not write to {bashrc}: {e}") - logger.info(f" Add manually: {export_line}") - logger.info(f" Current terminal: source ~/.bashrc (or open a new terminal)") - - -def _strip_derived_deploy_env() -> None: - """Drop derived TLS/auth env vars so the deploy pipeline decides solely from - cert-file presence in WEIGHTSLAB_CERTS_DIR (the single source of truth).""" - for key in _DERIVED_DEPLOY_ENV_VARS: - os.environ.pop(key, None) - - -def _banner() -> str: - """Return the WeightsLab ASCII banner, or a plain title if art is unavailable.""" - try: - from weightslab.art import _BANNER - return _BANNER - except Exception: - return "WeightsLab" - - -_DESCRIPTION = ( - _banner() - + "\nWeightsLab — Inspect, Edit, and Evolve Neural Networks\n" - + "Manage the Weights Studio UI, its Docker stack, and the secure " - + "(TLS + gRPC auth) environment." -) - -_EPILOG = """\ -commands: - se Set up the secure environment: generate TLS - certificates + a gRPC auth token in - ~/.weightslab-certs. Then set WEIGHTSLAB_CERTS_DIR - (the single source of truth) so the backend + new - shells find them. - --force-certs regenerate even if certs exist - - ui launch Purge stale weightslab/weights_studio Docker - resources, then build & start the UI stack. - UNSECURED (HTTP) by default — no certs generated. - --certs generate (if missing) + use TLS - certs + gRPC auth (HTTPS) - - start example Run a bundled PyTorch example (foreground; stop with - Ctrl+C). Installs the example's requirements first, - without prompting. Defaults to classification: - --cls classification example (default) - --seg segmentation example - --det detection example - --clus clustering example - --gen generation example - --3d_det 3D LiDAR point-cloud detection example - --2d_det 2D LiDAR point-cloud detection example - - cli Open an interactive terminal connected to a - currently-running experiment (pause/resume, status, - evaluate, agent query, etc.). Auto-discovers the - running experiment; the experiment must be serving - the CLI (e.g. wl.serve(serving_cli=True)). - --port PORT connect to a specific CLI port - --host HOST connect to a specific host (default: localhost) - -examples: - weightslab se # one-time secure setup (then export WEIGHTSLAB_CERTS_DIR) - weightslab se --force-certs # regenerate the certs - weightslab ui launch # clean + launch (unsecured HTTP, default) - weightslab ui launch --certs # secured launch (HTTPS + gRPC auth) - weightslab ui launch -i guillaumep2705/weightslab # pull frontend from a custom repo (latest) - weightslab ui launch -i guillaumep2705/weightslab -v v1.2.3 # pin a specific version/tag - weightslab start example # run the classification demo (default) - weightslab start example --seg # run the segmentation demo - weightslab start example --det # run the detection demo - weightslab start example --3d_det # run the 3D LiDAR detection demo - weightslab start example --2d_det # run the 2D LiDAR detection demo - weightslab cli # connect a terminal to the running experiment - weightslab cli --port 60000 # connect to a specific CLI port -""" - - -def _get_compose_file(): - """Return the path to the bundled docker-compose.yml.""" - # return files("weightslab.docker.docker") / "docker-compose.yml" - return Path(__file__).parent / 'docker' / 'docker' / 'docker-compose.yml' - - -def _get_envoy_config(): - """Return the path to the bundled envoy.yaml.""" - # return files("weightslab.docker.envoy") / "envoy.yaml" - return Path(__file__).parent / 'docker' / 'docker' / 'envoy.yaml' - - -def _get_bootstrap_script() -> Path: - """Get the bootstrap-secure.ps1 script path.""" - return Path(__file__).parent / 'docker' / 'docker' / 'utils' / 'build-and-deploy.sh' - - -def _get_cert_script() -> Path: - """Get the generate-certs-auth-token.sh script path.""" - return Path(__file__).parent / 'docker' / 'docker' / 'utils' / 'generate-certs-auth-token.sh' - - -def _get_cert_script_ps1() -> Path: - """Get the generate-certs-auth-token.ps1 script path.""" - return Path(__file__).parent / 'docker' / 'docker' / 'utils' / 'generate-certs-auth-token.ps1' - - -def _is_windows() -> bool: - """Check if running on Windows.""" - return sys.platform == 'win32' - - -def _make_executable(path) -> None: - """Add the execute bit to a file so it can be run directly (POSIX only). - - pip installs the bundled ``.sh`` scripts as package data *without* the execute - bit, and they are committed/shipped non-executable. ``_run_shell_script`` runs - them as a command (``bash -c "VAR=... '/path/script.sh'"``), which requires - the execute bit — otherwise bash fails with "Permission denied" until the user - ``chmod +x`` by hand. This grants u+x,g+x,o+x (e.g. 0644 -> 0755) and is a - best-effort no-op on Windows, where the POSIX execute bit is not used. - """ - if _is_windows(): - return - try: - mode = os.stat(path).st_mode - os.chmod(path, mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - except OSError as exc: - # Non-fatal: e.g. a root-owned system install the user can't chmod. They - # would not have been able to chmod it manually either; surface as debug. - logger.debug(f"Could not set execute bit on {path}: {exc}") - - -def _ensure_scripts_executable() -> None: - """Make every bundled shell script executable (POSIX only). - - Covers build-and-deploy.sh, generate-certs-auth-token.sh and the other ``.sh`` - files under ``weightslab/ui`` so a freshly pip-installed package can run them - without the user having to ``chmod +x`` first. No-op on Windows. - """ - if _is_windows(): - return - ui_dir = Path(__file__).parent / 'docker' - try: - scripts = list(ui_dir.rglob('*.sh')) - except OSError as exc: - logger.debug(f"Could not enumerate bundled scripts under {ui_dir}: {exc}") - return - for script in scripts: - _make_executable(script) - - -def _detect_compose_cmd(): - """Return the base command for Docker Compose, preferring v2 over v1. - - Docker Compose ships in two forms: - * v2 — the ``docker compose`` CLI plugin (bundled with Docker Desktop and - the recommended install). Invoked as two words: ``docker compose``. - * v1 — the legacy standalone ``docker-compose`` binary (one word, hyphen). - - We probe v2 first (``docker compose version``) then fall back to v1 - (``docker-compose version``) so ``weightslab ui launch`` works with either. - Returns ``["docker", "compose"]``, ``["docker-compose"]``, or None if neither - is available. - """ - # v2: the `docker compose` plugin. - try: - result = subprocess.run( - ["docker", "compose", "version"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - ) - if result.returncode == 0: - return ["docker", "compose"] - except FileNotFoundError: - # docker itself is missing; _check_docker() surfaces the user-facing error. - pass - - # v1: the legacy standalone `docker-compose` binary. - if shutil.which("docker-compose") is not None: - try: - result = subprocess.run( - ["docker-compose", "version"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - ) - if result.returncode == 0: - return ["docker-compose"] - except FileNotFoundError: - pass - - return None - - -def _compose_base_cmd(): - """Cached Docker Compose base command (v2 ``docker compose``, else v1 ``docker-compose``).""" - global _COMPOSE_BASE_CMD - if _COMPOSE_BASE_CMD is None: - _COMPOSE_BASE_CMD = _detect_compose_cmd() - return _COMPOSE_BASE_CMD - - -def _check_docker(): - """Verify that docker is installed and the daemon is running.""" - if shutil.which("docker") is None: - logger.error( - "Docker is required but not found on your PATH.\n" - "Install it from: https://docs.docker.com/get-docker/" - ) - sys.exit(1) - - try: - subprocess.run( - ["docker", "info"], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except subprocess.CalledProcessError: - logger.error( - "\n" - "=============================================================" - "=============================================================" - "Docker is installed but the daemon is not running.\n" - "Start it with: Docker Desktop or 'sudo systemctl start docker'." - "=============================================================" - "\n" - ) - sys.exit(1) - - -def _compose_cmd(compose_file, envoy_config, action): - """Build and run a docker compose command.""" - env = os.environ.copy() - env["WS_ENVOY_CONFIG"] = str(envoy_config) - - # If WEIGHTSLAB_CERTS_DIR isn't set, fall back to the default certs dir when - # it actually holds a full cert set + gRPC token. This gives docker compose a - # valid host-native bind-mount source (so Envoy gets its certs) even when the - # user never exported the var — and covers compose calls that run before - # ui_launch sets it (e.g. the stale-resource cleanup `compose down`). - if not env.get("WEIGHTSLAB_CERTS_DIR"): - default_mgr = CertAuthManager() - if default_mgr.has_valid_certs() and default_mgr.token_file.exists(): - host_path = _to_docker_host_path(default_mgr.certs_dir) - env["WEIGHTSLAB_CERTS_DIR"] = host_path - logger.info( - f"WEIGHTSLAB_CERTS_DIR not set — using default certs dir for docker: {host_path}" - ) - - base = _compose_base_cmd() - if base is None: - logger.error( - "Docker Compose is required but was not found.\n" - "Install Compose v2 (recommended) — the `docker compose` CLI plugin, " - "bundled with Docker Desktop: https://docs.docker.com/compose/install/\n" - "The legacy v1 `docker-compose` binary also works if it is on your PATH." - ) - sys.exit(1) - - action = list(action) - # Compose v1 (`docker-compose`) has no `up --pull ` flag (it is - # v2-only). Emulate it: `pull` first (best-effort — some services only build - # locally), then `up` without the flag. v2 supports it inline, so leave it. - if base == ["docker-compose"] and action and action[0] == "up" and "--pull" in action: - i = action.index("--pull") - del action[i:i + 2] # drop '--pull' and its policy value (e.g. 'always') - logger.info("Docker Compose v1 detected — pulling images before 'up'...") - pull_result = subprocess.run( - base + ["-f", str(compose_file), "pull"], - env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, - ) - if pull_result.stdout: - for line in pull_result.stdout.splitlines(): - logger.info(line) - - cmd = base + ["-f", str(compose_file)] + action - result = subprocess.run(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - - if result.stdout: - for line in result.stdout.splitlines(): - logger.info(line) - - if result.returncode != 0: - raise subprocess.CalledProcessError(result.returncode, cmd) - - -def _test_backend_connection(host: str = '127.0.0.1', port: int = 50051, timeout: float = 5.0) -> bool: - """Test if backend gRPC server is reachable.""" - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(timeout) - result = sock.connect_ex((host, port)) - sock.close() - return result == 0 - except Exception as e: - logger.debug(f"Backend connection test failed: {e}") - return False - - -def _run_powershell_script(script_path: str, args: list = None, env_vars: dict = None) -> int: - """Run a PowerShell script and return exit code.""" - if not _is_windows(): - logger.error("Secure launch requires Windows with PowerShell") - return 1 - - cmd = [ - 'powershell', - '-NoProfile', - '-ExecutionPolicy', 'Bypass', - '-File', script_path - ] - - if args: - cmd.extend(args) - - try: - env = os.environ.copy() - if env_vars: - env.update(env_vars) - result = subprocess.run(cmd, env=env) - return result.returncode - except Exception as e: - logger.error(f"Failed to run script: {e}") - return 1 - - -def _convert_to_git_bash_path(win_path: str) -> str: - """Convert Windows path to Git Bash compatible format.""" - # Normalize backslashes to forward slashes explicitly: Path(...).as_posix() - # does NOT convert '\' on POSIX hosts (e.g. Linux CI runners), so a Windows - # input path would keep its separators there. - p = str(win_path).replace("\\", "/") - # Convert C:/Users/... to /mnt/c/Users/... for Git Bash - if len(p) > 1 and p[1] == ':': - drive = p[0].lower() - rest = p[2:] - return f"/mnt/{drive}{rest}" - return p - - -def _to_docker_host_path(path) -> str: - """Return a path that Docker Desktop can bind-mount. - - On Windows, normalize WSL (/mnt/c/...) and Git Bash (/c/...) forms — and - native paths that Path() mangled into \\mnt\\c\\... — to C:/... form. - Docker Desktop cannot resolve /mnt-style sources and silently mounts an - empty directory, which crashes Envoy on missing certs. On Linux/macOS the - path is returned unchanged. - """ - p = str(path).replace('\\', '/') - if _is_windows(): - m = re.match(r'^/mnt/([a-zA-Z])/(.*)$', p) or re.match(r'^/([a-zA-Z])/(.*)$', p) - if m: - return f"{m.group(1).upper()}:/{m.group(2)}" - return p - - -def _run_shell_script(script_path: str, args: list = None, env_vars: dict = None) -> int: - """Run a shell script using bash with proper environment variable passing.""" - - try: - # Fix line endings in the file before running - with open(script_path, 'rb') as f: - script_bytes = f.read() - - # Ensure Unix line endings - fixed_bytes = script_bytes.replace(b'\r\n', b'\n').replace(b'\r', b'\n') - - # Write back if needed - if fixed_bytes != script_bytes: - with open(script_path, 'wb') as f: - f.write(fixed_bytes) - - # We invoke the script as a command below (bash -c "VAR=... 'script'"), - # which needs the execute bit. pip installs it without one, so add it here - # (no-op on Windows). Uses the on-disk path before any Git Bash conversion. - _make_executable(script_path) - - env = os.environ.copy() - if env_vars: - env.update(env_vars) - # Debug: log all environment variables being passed - for key, value in env_vars.items(): - logger.info(f"Passing env var: {key}={value}") - - # Build bash command - pass Windows path directly, script will handle conversion - # # Process path to ensure it's compatible with bash, especially on Windows - if _is_windows() and '\\' in script_path: - script_path = script_path.replace("\\", "/") # Ensure path is Unix-style for bash - script_path = _convert_to_git_bash_path(script_path) - logger.info(f"Converted script path for bash: {script_path}") - logger.info(f"Running shell script: {script_path} with args: {args} and env_vars: {env_vars}") - - # Build environment variable assignments for bash command - env_assignments = ' '.join([f"{k}='{v}'" for k, v in env_vars.items()]) if env_vars else "" - - if env_assignments: - # Pass env vars directly in bash command using -c flag - # This works on Windows (Git Bash), Linux, and macOS - logger.info('Using env assignments') - bash_script_cmd = f"{env_assignments} '{script_path}'" - if args: - bash_script_cmd += " " + " ".join(f"'{arg}'" for arg in args) - bash_cmd = ['bash', '-c', bash_script_cmd] - else: - logger.info('Not using env assignments') - bash_cmd = ['bash', '-x', str(script_path)] - if args: - bash_cmd.extend(args) - - result = subprocess.run(bash_cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - if result.stdout: - for line in result.stdout.splitlines(): - logger.info(line) - return result.returncode - except FileNotFoundError: - logger.error(f"Script file not found: {script_path}") - return 1 - except Exception as e: - logger.error(f"Failed to run script: {e}") - return 1 - - -def _generate_certs_with_fallback(force_certs: bool = False, certs_dir=None) -> int: - """Try shell script first, fall back to PowerShell on Windows if it fails. - - ``certs_dir`` is forwarded to the generation scripts as ``WEIGHTSLAB_CERTS_DIR`` - so certs land in the single source-of-truth directory (the scripts default to - ``~/.weightslab-certs`` when it is not provided). The shell script receives a - POSIX absolute path (``/mnt/c/...`` on Windows/WSL); PowerShell receives the - host-native path (``C:/...``). - """ - env_vars = None - if certs_dir is not None: - # Shell scripts (bash/WSL) need a POSIX-style absolute path. - # _to_docker_host_path gives C:/... which WSL bash treats as a relative - # path, creating the certs dir under cwd instead of ~/.weightslab-certs. - # Use the /mnt/c/... form so the path is absolute inside WSL. - if _is_windows(): - bash_certs_dir = _convert_to_git_bash_path(str(certs_dir)) - else: - bash_certs_dir = str(certs_dir) - env_vars = {'WEIGHTSLAB_CERTS_DIR': bash_certs_dir} - - cert_script = str(_get_cert_script()) - if not Path(cert_script).exists(): - logger.warning(f"Shell script not found: {cert_script}") - else: - script_args = [] - if force_certs: - script_args.append('--force-create-certs') - - logger.info("Attempting certificate generation with shell script...") - exit_code = _run_shell_script(cert_script, script_args, env_vars) - if exit_code == 0: - return 0 - logger.warning(f"Shell script failed (exit code {exit_code})") - - # Fallback to PowerShell on Windows - if _is_windows(): - logger.info("Falling back to PowerShell for certificate generation...") - cert_script_ps1 = str(_get_cert_script_ps1()) - if not Path(cert_script_ps1).exists(): - logger.error(f"PowerShell script not found: {cert_script_ps1}") - return 1 - - script_args = [] - if force_certs: - script_args.append('-ForceCreateCerts') - - exit_code = _run_powershell_script(cert_script_ps1, script_args, env_vars) - return exit_code - else: - logger.error("Neither shell nor PowerShell script could generate certificates") - return 1 - - -_DEV_CA_SUBJECT = "weightslab-dev-ca" - - -def _install_ca_trust(ca_file: Path) -> None: - """Install the dev CA into the OS trust store so browsers trust the HTTPS UI. - - Idempotent and safe to call on every launch. Platform behavior: - * Windows — adds to the CurrentUser\\Root store via the .NET X509Store API - (silent, no prompt). - * macOS — adds to the login keychain (may show a one-time auth prompt). - * Linux — installs into the system trust store via sudo (one-time prompt) - and, best-effort, the user's NSS DB so Chrome/Firefox trust it too. - - A failure here is non-fatal: TLS still works, the browser just shows a - self-signed warning until the CA is trusted manually. - """ - if not ca_file.exists(): - logger.warning(f"CA file not found, skipping trust install: {ca_file}") - return - - if _is_windows(): - ps = ( - "$ErrorActionPreference='Stop';" - f"$caPath='{ca_file}';" - "$certObj=New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($caPath);" - "$store=New-Object System.Security.Cryptography.X509Certificates.X509Store('Root','CurrentUser');" - "$store.Open('ReadWrite');" - "try{" - # Already trusted (same thumbprint)? Nothing to do — avoids a fragile Remove. - "$match=$store.Certificates|Where-Object{$_.Thumbprint -eq $certObj.Thumbprint};" - "if(-not $match){" - # Drop stale same-subject CAs from a previous rotation (best-effort). - f"$stale=$store.Certificates|Where-Object{{$_.Subject -eq 'CN={_DEV_CA_SUBJECT}'}};" - "foreach($c in $stale){try{$store.Remove($c)}catch{}};" - "$store.Add($certObj);" - "}" - "}finally{$store.Close()}" - ) - result = subprocess.run( - ["powershell", "-NoProfile", "-Command", ps], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - ) - if result.returncode == 0: - logger.info(" Dev CA trusted in Windows CurrentUser\\Root store (restart browser to apply)") - else: - logger.warning(f"Could not auto-trust dev CA: {result.stderr.strip()}") - return - - if sys.platform == "darwin": - check = subprocess.run( - ["security", "find-certificate", "-c", _DEV_CA_SUBJECT], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - ) - if check.returncode == 0: - logger.info(" Dev CA already trusted (macOS keychain)") - return - logger.info("Installing dev CA into macOS login keychain (may prompt)...") - subprocess.run( - ["security", "add-trusted-cert", "-r", "trustRoot", - "-k", os.path.expanduser("~/Library/Keychains/login.keychain-db"), str(ca_file)], - ) - return - - # Linux: system trust store (curl/openssl) + best-effort NSS (browsers). - system_ca = Path("/usr/local/share/ca-certificates/weightslab-dev-ca.crt") - try: - already = system_ca.exists() and system_ca.read_bytes() == ca_file.read_bytes() - except OSError: - already = False - if not already: - logger.info("Installing dev CA into the Linux system trust store (may prompt for sudo)...") - subprocess.run(["sudo", "cp", str(ca_file), str(system_ca)]) - subprocess.run(["sudo", "update-ca-certificates"]) - else: - logger.info(" Dev CA already in Linux system trust store") - - # Browsers use their own NSS DB; add it there too if certutil is available. - if shutil.which("certutil"): - nssdb = Path.home() / ".pki" / "nssdb" - nssdb.mkdir(parents=True, exist_ok=True) - subprocess.run( - ["certutil", "-A", "-n", _DEV_CA_SUBJECT, "-t", "C,,", - "-i", str(ca_file), "-d", f"sql:{nssdb}"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - ) - - -def _ensure_certificates(manager: CertAuthManager, force_certs: bool = False) -> bool: - """Generate certs + auth token in ``manager.certs_dir`` if missing (or forced). - - Does NOT export any TLS/auth env vars: the launch pipeline derives TLS purely - from cert-file presence in ``WEIGHTSLAB_CERTS_DIR`` (the single source of - truth). Returns True if certs are present afterwards, False otherwise. - """ - if manager.has_any_credentials() and not force_certs: - logger.info(f" Using existing credentials in {manager.certs_dir}") - manager.get_or_create_auth_token() - # Ensure the CA is trusted even when reusing certs from a prior run that - # was generated via bash (which does not install OS trust). - _install_ca_trust(manager.ca_file) - return manager.has_valid_certs() - - logger.info( - "Regenerating certificates (--force-certs)..." - if force_certs - else "No certificates found — generating them now..." - ) - Path(manager.certs_dir).mkdir(parents=True, exist_ok=True) - exit_code = _generate_certs_with_fallback(force_certs=force_certs, certs_dir=manager.certs_dir) - if exit_code != 0: - logger.warning("Certificate generation failed — continuing in unsecured mode") - return False - - manager.get_or_create_auth_token() - _install_ca_trust(manager.ca_file) - logger.info(f" Certificates ready in {manager.certs_dir}") - return manager.has_valid_certs() - - -def _remove_docker_image(image: str) -> None: - """Remove a Docker image (all tags) if present; no-op when absent.""" - result = subprocess.run( - ["docker", "images", "-q", image], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, - ) - image_ids = sorted(set(result.stdout.split())) - if not image_ids: - return - logger.info(f"Removing cached image '{image}' ({len(image_ids)} ref(s))...") - subprocess.run( - ["docker", "rmi", "-f", *image_ids], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - ) - - -def _clean_stale_docker_resources(frontend_image: str = _FRONTEND_IMAGE) -> None: - """Remove leftover weightslab / weights_studio Docker state before a launch. - - Stale containers, the compose network, anonymous volumes, a cached frontend - image, and a leftover generated ``.env`` can each silently break a fresh - launch (an old image served instead of a rebuild, or an empty cert mount - that crashes Envoy). This is scoped STRICTLY to weightslab/weights_studio - resources — it never runs a global ``docker system prune``. - - ``frontend_image`` is the repo (optionally ``repo:tag``) whose cached copy is - dropped so ``up --pull always`` fetches a fresh one — pass the resolved value - when ``--image/--version`` point at a non-default repo. - """ - logger.info("Cleaning stale Docker resources (weightslab/weights_studio only)...") - compose_file = _get_compose_file() - envoy_config = _get_envoy_config() - - # 1. Tear down any existing stack: containers + default network + anon volumes. - try: - _compose_cmd(compose_file, envoy_config, ["down", "--remove-orphans", "--volumes"]) - except subprocess.CalledProcessError as exc: - logger.debug(f"'compose down' returned non-zero (nothing to remove?): {exc}") - - # 2. Force-remove leftover named containers started outside this compose project. - for container in _STACK_CONTAINERS: - subprocess.run( - ["docker", "rm", "-f", container], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - ) - - # 3. Drop the cached frontend image so a fresh one is built/pulled. - _remove_docker_image(frontend_image) - - # 4. Remove the generated .env so stale values don't leak into compose. - env_file = Path(compose_file).parent / ".env" - if env_file.exists(): - try: - env_file.unlink() - logger.info(f"Removed stale env file: {env_file}") - except OSError as exc: - logger.debug(f"Could not remove {env_file}: {exc}") - - -def ui_launch(args): - """Purge stale Docker state, then build & start the UI. - - By default the UI launches UNSECURED (HTTP, no gRPC auth) — no certificates - are generated. Pass ``--certs`` to generate (if missing) and use TLS certs + - a gRPC auth token. Existing certs in WEIGHTSLAB_CERTS_DIR are always honored - (file presence is the single source of truth) and are never deleted here. - - Flags (all optional, read defensively so legacy callers still work): - --certs generate (if missing) and use TLS certs + gRPC auth (HTTPS) - --force-certs with --certs, regenerate certificates even if they exist - --no-clean skip the stale Docker resource cleanup step - --dev use the dev compose overlay - -i/--image frontend image repo to run/pull (default: graybx/weightslab) - -v/--version frontend image tag/version to pull (default: latest) - """ - try: - from weightslab.utils.telemetry import ping_ui_launch - from weightslab import __version__ as _wl_version - ping_ui_launch(_wl_version) - except Exception: - pass - _check_docker() - # pip installs the bundled .sh scripts without the execute bit; make them - # runnable so the user never has to `chmod +x` before `weightslab ui launch`. - _ensure_scripts_executable() - - use_certs = getattr(args, "certs", False) - no_auth = getattr(args, "no_auth", False) - force_certs = getattr(args, "force_certs", False) - no_clean = getattr(args, "no_clean", False) - certs_dir_arg = getattr(args, "certs_dir", None) - - # Resolve which frontend image to run. docker compose reads it via the - # WS_FRONTEND_IMAGE env var (see `${WS_FRONTEND_IMAGE:-...}` in the compose - # file); export it so both `up --pull always` below and the stale-image - # cleanup target the same repo/tag. - frontend_image = _resolve_frontend_image( - getattr(args, "image", None), getattr(args, "version", None) - ) - os.environ["WS_FRONTEND_IMAGE"] = frontend_image - frontend_repo = frontend_image.rsplit(":", 1)[0] if ":" in frontend_image.rsplit("/", 1)[-1] else frontend_image - logger.info(f"Using frontend image: {frontend_image}") - - # A custom certs dir (positional arg) takes precedence over the env var / - # default. Otherwise fall back to $WEIGHTSLAB_CERTS_DIR or ~/.weightslab-certs. - if certs_dir_arg: - # Resolve to an absolute path so Windows Python, WSL bash and docker all - # agree on the location (a relative path resolves differently per shell). - certs_dir_arg = str(Path(certs_dir_arg).resolve()) - manager = CertAuthManager(certs_dir=certs_dir_arg, enable_auth=not no_auth) - logger.info(f"Using custom certs directory: {manager.certs_dir}") - else: - manager = CertAuthManager.from_env_or_default(enable_auth=not no_auth) - - if use_certs: - _ensure_certificates(manager, force_certs=force_certs) - else: - # Default: do NOT generate certs. Existing certs in WEIGHTSLAB_CERTS_DIR - # are still honored (file-presence is the single source of truth); this - # never deletes anything. Ensure the dir exists for the compose bind-mount. - logger.info("Launching WITHOUT cert generation (default; HTTP). " - "Pass --certs for secured HTTPS + gRPC auth.") - try: - manager.certs_dir.mkdir(parents=True, exist_ok=True) - except (OSError, AttributeError): - # AttributeError: certs_dir is a plain str (e.g. in unit tests). - try: - os.makedirs(str(manager.certs_dir), exist_ok=True) - except OSError: - logger.warning('Fail to create weightslab-certs directory!') - - # Single source of truth: TLS is on iff cert files exist in the dir. - secured = manager.has_valid_certs() - - # Set the correct certs path in the process env NOW so that _compose_cmd - # passes it to docker compose — this overrides any stale/malformed value - # in a leftover .env file and prevents the "too many colons" bind-mount error. - os.environ["WEIGHTSLAB_CERTS_DIR"] = _to_docker_host_path(manager.certs_dir) - - # Remove stale weightslab/weights_studio Docker resources that could prevent - # a clean launch. Scoped strictly to our own resources (see helper docstring). - if no_clean: - logger.info("Skipping stale Docker resource cleanup (--no-clean)") - else: - _clean_stale_docker_resources(frontend_repo) - - # Single source of truth: the deploy pipeline (build-and-deploy.sh + the - # compose `auto` logic) derives TLS/auth solely from cert-file presence in - # WEIGHTSLAB_CERTS_DIR. Strip any pre-set/derived TLS env so it cannot - # override that file-based decision. - _strip_derived_deploy_env() - - # Run bootstrap script to setup environment - bootstrap_script = str(_get_bootstrap_script()) - if Path(bootstrap_script).exists(): - logger.info("Running bootstrap script...") - logger.info(f"Bootstrap script path: {bootstrap_script}") - # Convert Windows path to Unix-style for bash - certs_dir_str = str(manager.certs_dir) - if _is_windows() and '\\' in certs_dir_str: - certs_dir_str = _convert_to_git_bash_path(certs_dir_str) - logger.info(f"Converted path to Unix-style: {certs_dir_str}") - - # Calculate WEIGHTSLAB_ROOT (parent of weightslab package) - weightslab_root = str(Path(__file__).parent) - if _is_windows() and '\\' in weightslab_root: - weightslab_root = _convert_to_git_bash_path(weightslab_root) - - # Docker Desktop (Windows) bind mounts need a host-native path - # (e.g. C:/Users/...), NOT the /mnt/c Unix path used for the bash - # script's own file checks. as_posix() yields C:/... on Windows and a - # normal /abs/path on Linux/macOS — correct for docker compose on every - # platform. The bash script writes this into .env for the compose mount. - certs_dir_host = _to_docker_host_path(manager.certs_dir) - - # Always pass the real certs dir so the compose bind-mount has a valid - # source. When there are no certs we add --unsecure, which forces - # Envoy/nginx to plaintext (mounted files, if any, are ignored) without - # leaving the mount source empty. - bootstrap_env_vars = { - 'WEIGHTSLAB_CERTS_DIR': certs_dir_str, - 'WEIGHTSLAB_CERTS_DIR_HOST': certs_dir_host, - 'WEIGHTSLAB_ROOT': weightslab_root - } - # On Windows the bootstrap runs in WSL/Git-Bash where `docker` usually - # isn't on PATH; the compose up below (run from Windows Python) handles - # the deploy. Tell the script to only write .env and skip docker ops so - # it doesn't emit a spurious "build failed" error. - if _is_windows(): - bootstrap_env_vars['WEIGHTSLAB_SKIP_DOCKER_OPS'] = '1' - script_args = [] - if not secured: - script_args.append('--unsecure') - if getattr(args, "dev", False): - script_args.append('--dev') - logger.info(f"WEIGHTSLAB_CERTS_DIR={bootstrap_env_vars['WEIGHTSLAB_CERTS_DIR']}") - logger.info(f"WEIGHTSLAB_CERTS_DIR_HOST={bootstrap_env_vars['WEIGHTSLAB_CERTS_DIR_HOST']}") - logger.info(f"WEIGHTSLAB_ROOT={bootstrap_env_vars['WEIGHTSLAB_ROOT']}") - exit_code = _run_shell_script(bootstrap_script, script_args, bootstrap_env_vars) - if exit_code != 0: - logger.warning(f"Bootstrap script exited with code {exit_code}, continuing anyway...") - else: - logger.warning(f"Bootstrap script not found: {bootstrap_script}") - - # docker compose gives an exported env var precedence over .env. Force the - # host-native certs path here (on every path, secured or not) so our - # compose call below bind-mounts a real folder — a stray /mnt/c or empty - # value would mount an empty/invalid dir and can crash Envoy. - os.environ["WEIGHTSLAB_CERTS_DIR"] = _to_docker_host_path(manager.certs_dir) - - _compose_cmd( - _get_compose_file(), - _get_envoy_config(), - ["up", "-d", "--pull", "always"], - ) - port = os.environ.get("VITE_PORT", "5173") - # HTTPS iff cert files actually exist in WEIGHTSLAB_CERTS_DIR — the same - # file-presence rule the deploy pipeline applies. - secured = manager.has_valid_certs() - protocol = "https" if secured else "http" - logger.info(f"Weights Studio UI is running at: {protocol}://localhost:{port}") - if secured: - certs_dir_str = str(manager.certs_dir) - # Persist first (it logs its own lines), so the export/setx reminder - # below stays the FINAL output and can't be missed. - # Persist when a custom dir was given (it won't match the default, so the - # backend must be told where to look) or when the var isn't already set. - if certs_dir_arg or not _CERTS_DIR_IN_ORIGINAL_ENV: - _persist_certs_dir(certs_dir_str) - # The backend and any new shell must point at the same certs dir, or - # they'll mismatch the UI's TLS/auth. Keep this the last thing printed. - logger.warning("") - logger.warning(" ACTION REQUIRED — TLS is ON. Set WEIGHTSLAB_CERTS_DIR so the " - "training backend and new terminals use the same certificates:") - logger.warning(f" (bash) export WEIGHTSLAB_CERTS_DIR=\"{certs_dir_str}\"") - logger.warning(f" (Windows) setx WEIGHTSLAB_CERTS_DIR \"{certs_dir_str}\"") - else: - logger.info("UI is running UNSECURED (HTTP, no gRPC auth). " - "Re-run with `weightslab ui launch --certs` for TLS.") - - -def ui_secure_environment(args): - """`weightslab se`: create a certs directory with certs + gRPC token. - - The directory is the single source of truth — WEIGHTSLAB_CERTS_DIR is exported - for this process and the user is asked to export it globally. Everything else - (TLS on/off, auth on/off) is derived from the files in that directory by the - backend and the deploy pipeline, so this command does not set any other env. - """ - logger.info("Setting up secure environment...") - # Bundled .sh scripts ship without the execute bit (pip strips it); make them - # runnable so the cert-generation script below doesn't fail on "Permission denied". - _ensure_scripts_executable() - - force_certs = getattr(args, "force_certs", False) - no_auth = getattr(args, "no_auth", False) - certs_dir = getattr(args, "certs_dir", None) - if certs_dir: - # Absolute path so Windows Python, WSL bash and docker agree on location. - certs_dir = str(Path(certs_dir).resolve()) - - # Resolve the target directory first (no filesystem work in __init__), so we - # can point the generation scripts at it via WEIGHTSLAB_CERTS_DIR. - manager = CertAuthManager(certs_dir=certs_dir, enable_auth=not no_auth) - - exit_code = _generate_certs_with_fallback(force_certs=force_certs, certs_dir=manager.certs_dir) - if exit_code != 0: - logger.error("Certificate generation failed") - sys.exit(1) - - manager.certs_dir.mkdir(parents=True, exist_ok=True) - manager.get_or_create_auth_token() - - # Export ONLY the single source of truth for this process. - os.environ["WEIGHTSLAB_CERTS_DIR"] = str(manager.certs_dir) - - logger.info(" Certificates generated successfully") - logger.info(" gRPC auth token created") - logger.info(f" Certs and token stored in: {manager.certs_dir}") - logger.info(f" WEIGHTSLAB_CERTS_DIR exported for this process: {manager.certs_dir}") - logger.info("Then launch the secured UI with: weightslab ui launch --certs") - # Keep this the FINAL output so the user can't miss the action they must take. - logger.warning("") - logger.warning(" ACTION REQUIRED — set WEIGHTSLAB_CERTS_DIR globally so new shells " - "and the training backend find these certs (single source of truth):") - logger.warning(f" (bash) echo 'export WEIGHTSLAB_CERTS_DIR=\"{manager.certs_dir}\"' >> ~/.bashrc && source ~/.bashrc") - logger.warning(f" (Windows) setx WEIGHTSLAB_CERTS_DIR \"{manager.certs_dir}\"") - - -# Bundled PyTorch examples, keyed by the CLI flag (e.g. --cls -> ws-classification). -# Each value is (directory name under examples/PyTorch, human-readable label). -# kind -> (dir_name, label, category) where category is the examples/ subfolder. -_EXAMPLES = { - "cls": ("ws-classification", "classification", "PyTorch"), - "seg": ("ws-segmentation", "segmentation", "PyTorch"), - "det": ("ws-detection", "detection", "PyTorch"), - "clus": ("ws-clustering", "clustering", "PyTorch"), - "gen": ("ws-generation", "generation", "PyTorch"), - "3d_det": ("ws-3d-lidar-detection", "3D LiDAR detection", "Usecases"), - "2d_det": ("ws-2d-lidar-detection", "2D LiDAR detection", "Usecases"), -} -_DEFAULT_EXAMPLE = "cls" - - -def _get_example_dir(name: str = "ws-classification", category: str = "PyTorch") -> Path: - """Path to a bundled example directory (under examples//).""" - return Path(__file__).parent / 'examples' / category / name - - -def _install_example_requirements(example_dir: Path) -> None: - """Install an example's requirements non-interactively, if a file is present. - - Looks for requirements.txt (then requirements.in) in the example directory and - runs `pip install -r` with the current interpreter and `--no-input` so it never - prompts. Non-fatal: a failure is logged and the example is still attempted, so a - transient install hiccup doesn't block a run where deps are already satisfied. - """ - for fname in ("requirements.txt", "requirements.in"): - req = example_dir / fname - if not req.exists(): - continue - logger.info(f"Installing example requirements (non-interactive): {req}") - try: - subprocess.run( - [sys.executable, "-m", "pip", "install", "-r", str(req), - "--no-input", "--disable-pip-version-check"], - check=True, - ) - except subprocess.CalledProcessError as exc: - logger.warning( - f"Failed to install requirements ({req}): {exc}. " - "Continuing — the example may still run if deps are already installed." - ) - return # only the first matching requirements file is used - - -def example_start(args): - """`weightslab start example [--cls|--seg|--clus|--gen]`: run a bundled example. - - Defaults to the classification (cls) example. First installs the example's - requirements (if a requirements file is present) without prompting, then runs - its main.py with the current Python interpreter from its own directory so it - resolves its sibling config.yaml. Runs in the foreground (serves until Ctrl+C). - """ - kind = getattr(args, "example_kind", None) or _DEFAULT_EXAMPLE - dir_name, label, category = _EXAMPLES.get(kind, _EXAMPLES[_DEFAULT_EXAMPLE]) - - example_dir = _get_example_dir(dir_name, category) - main_py = example_dir / "main.py" - if not main_py.exists(): - logger.error(f"{label.capitalize()} example not found: {main_py}") - sys.exit(1) - - # Install the example's own requirements first, without any interaction. - _install_example_requirements(example_dir) - - logger.info(f"Starting the WeightsLab {label} ({kind}) example...") - logger.info(f" {main_py}") - logger.info("In another terminal, launch the UI with: weightslab ui launch") - logger.info(f"Then open http://localhost:5173 — stop the example with Ctrl+C.") - if not _CERTS_DIR_IN_ORIGINAL_ENV: - manager = CertAuthManager.from_env_or_default() - if manager.has_valid_certs(): - logger.warning( - "WEIGHTSLAB_CERTS_DIR is not set in your shell environment. " - "TLS will work this session (certs found at default location) " - "but may not persist across terminals." - ) - _persist_certs_dir(str(manager.certs_dir)) - try: - env = os.environ.copy() - env['WEIGHTSLAB_SUPPRESS_BANNER'] = '1' - result = subprocess.run([sys.executable, str(main_py)], cwd=str(example_dir), env=env) - except KeyboardInterrupt: - logger.info("Example stopped.") - return - if result.returncode != 0: - sys.exit(result.returncode) - - -def cli_connect(args): - """`weightslab cli [--port N] [--host H]`: open an interactive terminal - connected to a currently-running experiment's CLI server. - - With no --port, auto-discovers the running experiment (the backend advertises - its actual port on startup). Pass --port to target a specific server. - """ - try: - import weightslab.backend.cli as cli_backend - except Exception as exc: - logger.error(f"Could not load the WeightsLab CLI client: {exc}") - sys.exit(1) - - port = getattr(args, "port", None) - host = getattr(args, "host", None) - exit_code = cli_backend.cli_connect(cli_port=port, cli_host=host) - sys.exit(exit_code) - - -def _add_example_kind_flags(p: argparse.ArgumentParser) -> None: - """Attach the mutually-exclusive example-kind flags (default: classification).""" - group = p.add_mutually_exclusive_group() - group.add_argument("--cls", action="store_const", dest="example_kind", const="cls", - help="Run the classification example (default)") - group.add_argument("--seg", action="store_const", dest="example_kind", const="seg", - help="Run the segmentation example") - group.add_argument("--det", action="store_const", dest="example_kind", const="det", - help="Run the detection example") - group.add_argument("--clus", action="store_const", dest="example_kind", const="clus", - help="Run the clustering example") - group.add_argument("--gen", action="store_const", dest="example_kind", const="gen", - help="Run the generation example") - group.add_argument("--3d_det", action="store_const", dest="example_kind", const="3d_det", - help="Run the 3D LiDAR point-cloud detection example") - group.add_argument("--2d_det", action="store_const", dest="example_kind", const="2d_det", - help="Run the 2D LiDAR point-cloud detection example") - p.set_defaults(example_kind=_DEFAULT_EXAMPLE) - - -def _build_parser() -> argparse.ArgumentParser: - """Build the top-level argument parser (banner + detailed command reference). - - The CLI is intentionally minimal — exactly these commands: - weightslab --help | -h | help - weightslab se [--force-certs] - weightslab ui launch [--certs] - weightslab start example [--cls|--seg|--det|--clus|--gen|--3d_det|--2d_det] - weightslab cli [--port PORT] [--host HOST] - """ - parser = argparse.ArgumentParser( - prog="weightslab", - description=_DESCRIPTION, - epilog=_EPILOG, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - # metavar lists only the documented commands; the `example` alias is accepted - # but intentionally omitted here (and help=SUPPRESS'd below) so it stays hidden. - sub = parser.add_subparsers(dest="command", metavar="{se,ui,start,cli,help}") - - # weightslab se [--force-certs] [certs_dir] - se_parser = sub.add_parser("se", help="Set up the secure environment (TLS certs + gRPC auth token)") - se_parser.add_argument('--force-certs', action='store_true', help='Regenerate certificates even if they already exist') - se_parser.add_argument('certs_dir', nargs='?', default=None, - help='Custom directory for certs/token (default: $WEIGHTSLAB_CERTS_DIR or ~/.weightslab-certs)') - - # weightslab ui launch [--certs] [certs_dir] - ui_parser = sub.add_parser("ui", help="Manage the Weights Studio UI") - ui_sub = ui_parser.add_subparsers(dest="action") - launch_ui_parser = ui_sub.add_parser( - "launch", help="Clean stale Docker state, then launch the UI (unsecured by default; --certs for TLS)") - launch_ui_parser.add_argument('--certs', action='store_true', help='Generate (if missing) and use TLS certs + gRPC auth token (secured HTTPS). Default: unsecured HTTP.') - launch_ui_parser.add_argument('-i', '--image', default=None, - help='Frontend image repo to run/pull (default: graybx/weightslab). ' - 'e.g. guillaumep2705/weightslab') - launch_ui_parser.add_argument('-v', '--version', default=None, - help='Frontend image tag/version to pull (default: latest). e.g. v1.2.3') - launch_ui_parser.add_argument('certs_dir', nargs='?', default=None, - help='Custom directory for certs/token (default: $WEIGHTSLAB_CERTS_DIR or ~/.weightslab-certs)') - - # weightslab cli [--port N] [--host H] - cli_parser = sub.add_parser( - "cli", help="Open an interactive terminal connected to the running experiment") - cli_parser.add_argument('--port', type=int, default=None, - help='CLI server port (default: auto-discover the running experiment)') - cli_parser.add_argument('--host', default=None, - help='CLI server host (default: localhost)') - - # weightslab start example [--cls|--seg|--clus|--gen] - start_parser = sub.add_parser("start", help="Start a bundled WeightsLab resource") - start_sub = start_parser.add_subparsers(dest="start_target") - example_parser = start_sub.add_parser( - "example", help="Start a bundled PyTorch example (default: classification)") - _add_example_kind_flags(example_parser) - - # Tolerate the swapped order: `weightslab example start [flags]` (and bare - # `weightslab example`) behave exactly like `weightslab start example`. Hidden - # from --help on purpose (argparse.SUPPRESS) — it's a forgiving fallback, not a - # documented command. - example_alias = sub.add_parser("example", help=argparse.SUPPRESS) - example_alias_sub = example_alias.add_subparsers(dest="example_action") - example_alias_start = example_alias_sub.add_parser( - "start", help="Start a bundled PyTorch example (default: classification)") - _add_example_kind_flags(example_alias_start) - - sub.add_parser("help", help="Show this help message") - - return parser, ui_parser, start_parser, cli_parser - - -def main(): - parser, ui_parser, start_parser, cli_parser = _build_parser() - args = parser.parse_args() - - if args.command == "help" or args.command is None: - parser.print_help() - elif args.command == "cli": - cli_connect(args) - elif args.command == "se": - ui_secure_environment(args) - elif args.command == "ui": - if getattr(args, "action", None) == "launch": - ui_launch(args) - else: - ui_parser.print_help() - elif args.command == "start": - if getattr(args, "start_target", None) == "example": - example_start(args) - else: - start_parser.print_help() - elif args.command == "example": - # Alias for `start example` — tolerate the swapped subcommand order - # (`weightslab example start [flags]`) and the bare `weightslab example`. - example_start(args) - else: - parser.print_help() - - -if __name__ == "__main__": - main() diff --git a/weightslab/utils/__init__.py b/weightslab/utils/__init__.py index af11e464..6131e7c0 100644 --- a/weightslab/utils/__init__.py +++ b/weightslab/utils/__init__.py @@ -1,9 +1,36 @@ -from .tools import filter_kwargs_for_callable, safe_call_with_kwargs, capture_rng_state, restore_rng_state, recursive_update - -__all__ = [ - 'filter_kwargs_for_callable', - 'safe_call_with_kwargs', - 'capture_rng_state', - 'restore_rng_state', - 'recursive_update', -] +"""weightslab.utils package. + +The convenience re-exports from `.tools` are resolved lazily (PEP 562): `.tools` +imports torch/numpy, so eager re-export here would pull the heavy stack into +every `weightslab.utils.` import (e.g. the torch-free `weightslab.utils.logs` +used by the UI/CLI). Names still import normally on first access, e.g. +``from weightslab.utils import filter_kwargs_for_callable``. +""" +import importlib + +_LAZY_EXPORTS = { + name: ".tools" + for name in ( + "filter_kwargs_for_callable", + "safe_call_with_kwargs", + "capture_rng_state", + "restore_rng_state", + "recursive_update", + ) +} + +__all__ = list(_LAZY_EXPORTS) + + +def __getattr__(name): # PEP 562 + module_name = _LAZY_EXPORTS.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module = importlib.import_module(module_name, __name__) + value = getattr(module, name) + globals()[name] = value + return value + + +def __dir__(): + return sorted(set(globals()) | set(_LAZY_EXPORTS)) diff --git a/weightslab/utils/telemetry.py b/weightslab/utils/telemetry.py index 57c5fb27..8b2f2e03 100644 --- a/weightslab/utils/telemetry.py +++ b/weightslab/utils/telemetry.py @@ -1,4 +1,4 @@ -"""Anonymous usage telemetry — fired on first daily import and on `weightslab ui launch`. +"""Anonymous usage telemetry — fired on first daily import and on `weightslab start`. Opt-out: set WL_NO_TELEMETRY=1 in your environment. No personally identifiable information is collected. The raw IP is used @@ -87,7 +87,7 @@ def _payload(event: str, version: str) -> bytes: tz = "unknown" return json.dumps({ "uuid": _get_or_create_uuid(), - "event": event, # "import" | "ui_launch" + "event": event, # "import" | "ui_start" "version": version, "python": sys.version.split()[0], "os": platform.system(), # Windows / Linux / Darwin @@ -138,15 +138,15 @@ def ping_import(version: str) -> None: def ping_ui_launch(version: str) -> None: - """Async ping on `weightslab ui launch` — at most once per 24 h or version change. No-op in CI.""" + """Async ping on `weightslab start` — at most once per 24 h or version change. No-op in CI.""" if _disabled() or _is_ci(): return if not _ping_due(_LAST_UI_PING_FILE, version): - logger.debug("Telemetry ui_launch ping skipped (24 h cooldown active)") + logger.debug("Telemetry ui_start ping skipped (24 h cooldown active)") return _record_ping(_LAST_UI_PING_FILE, version) logger.info( "WeightsLab uses anonymous usage data (package version used, and OS name). " "Set WL_NO_TELEMETRY=1 to disable." ) - _fire("ui_launch", version) + _fire("ui_start", version) diff --git a/weightslab/utils/tools.py b/weightslab/utils/tools.py index b57ff83a..f8f25e54 100644 --- a/weightslab/utils/tools.py +++ b/weightslab/utils/tools.py @@ -1,4 +1,5 @@ import io +import sys import xxhash import types import logging @@ -22,6 +23,23 @@ # -------------------------- Utils Functions --------------------------------- # ---------------------------------------------------------------------------- +def _running_in_notebook() -> bool: + """True when running inside a Jupyter notebook kernel or Google Colab. + + Distinguishes a notebook kernel (``ZMQInteractiveShell``) / Colab from a + plain script or a terminal IPython session, so we only nudge users who can't + reach a locally-launched Weights Studio without a tunnel. + """ + if "google.colab" in sys.modules: + return True + try: + from IPython import get_ipython + shell = get_ipython() + return shell is not None and shell.__class__.__name__ == "ZMQInteractiveShell" + except Exception: + return False + + def safe_reset_index(df: "pd.DataFrame") -> "pd.DataFrame": """Reset DataFrame index levels into columns, skipping any level whose name is already a column.