diff --git a/Cargo.lock b/Cargo.lock index 9871402b31..35e4d1954e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -629,7 +629,7 @@ dependencies = [ [[package]] name = "dimos-lcm" version = "0.1.0" -source = "git+https://github.com/dimensionalOS/dimos-lcm.git?branch=rust-codegen#04d78e8622500244123ba9cefa4c51b4cb454549" +source = "git+https://github.com/dimensionalOS/dimos-lcm.git?rev=dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5#dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5" dependencies = [ "byteorder", "socket2 0.5.10", @@ -644,7 +644,7 @@ dependencies = [ "crossbeam-channel", "crossbeam-utils", "dimos-module", - "lcm-msgs", + "lcm-msgs 0.1.0 (git+https://github.com/dimensionalOS/dimos-lcm.git?branch=rust-codegen)", "lz4_flex 0.14.0", "mcap", "rayon", @@ -664,7 +664,7 @@ version = "0.1.0" dependencies = [ "ahash", "dimos-module", - "lcm-msgs", + "lcm-msgs 0.1.0 (git+https://github.com/dimensionalOS/dimos-lcm.git?rev=dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5)", "rayon", "serde", "tokio", @@ -690,7 +690,7 @@ version = "0.1.0" dependencies = [ "dimos-lcm", "dimos-module-macros", - "lcm-msgs", + "lcm-msgs 0.1.0 (git+https://github.com/dimensionalOS/dimos-lcm.git?rev=dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5)", "nalgebra", "rayon", "serde", @@ -711,7 +711,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.4", - "toml 1.1.4+spec-1.1.0", + "toml 1.1.5+spec-1.1.0", ] [[package]] @@ -719,7 +719,7 @@ name = "dimos-native-module-examples" version = "0.1.0" dependencies = [ "dimos-module", - "lcm-msgs", + "lcm-msgs 0.1.0 (git+https://github.com/dimensionalOS/dimos-lcm.git?rev=dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5)", "serde", "tokio", "tracing", @@ -733,7 +733,7 @@ dependencies = [ "ahash", "arrayvec", "dimos-module", - "lcm-msgs", + "lcm-msgs 0.1.0 (git+https://github.com/dimensionalOS/dimos-lcm.git?rev=dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5)", "nalgebra", "rayon", "serde", @@ -1604,6 +1604,14 @@ dependencies = [ "byteorder", ] +[[package]] +name = "lcm-msgs" +version = "0.1.0" +source = "git+https://github.com/dimensionalOS/dimos-lcm.git?rev=dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5#dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5" +dependencies = [ + "byteorder", +] + [[package]] name = "libc" version = "0.2.189" @@ -1628,9 +1636,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96" +checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" dependencies = [ "libc", ] @@ -1776,9 +1784,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi", @@ -2290,13 +2298,13 @@ dependencies = [ [[package]] name = "prometheus-client-derive-encode" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9adf1691c04c0a5ff46ff8f262b58beb07b0dbb61f96f9f54f6cbd82106ed87f" +checksum = "01e34894696ff94f64a20c2c373a6440903e9c2789a303d68ec6e6f953f890e4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] @@ -3128,9 +3136,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" [[package]] name = "socket2" @@ -3439,9 +3447,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" dependencies = [ "tinyvec_macros", ] @@ -3521,9 +3529,9 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.4+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +checksum = "12c0ba9680044b4ce98d391a62094047eada0d64860b80166c39f4a6b5640785" dependencies = [ "indexmap 2.14.1", "serde_core", diff --git a/dimos/cli/commands/collect.py b/dimos/cli/commands/collect.py new file mode 100644 index 0000000000..71b6698abd --- /dev/null +++ b/dimos/cli/commands/collect.py @@ -0,0 +1,343 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Interactive controls for an already-running teach collection stack.""" + +from __future__ import annotations + +import time +from typing import Any, cast + +from textual.app import App, ComposeResult +from textual.binding import Binding +from textual.containers import Container, Horizontal +from textual.widgets import Button, Footer, Static +import typer + +from dimos.cli import theme +from dimos.msgs.imitation_msgs.EpisodeStatus import EpisodeStatus +from dimos.porcelain.dimos import Dimos + +_MONITOR = "EpisodeMonitorModule" +_COORDINATOR = "ControlCoordinator" +_REQUIRED_TASKS = {"teach_openyam"} + + +class TeachCollectionSession: + """RPC client for the operator controls used by the collection panel.""" + + def __init__(self, client: Dimos, monitor: Any) -> None: + self._client = client + self._monitor = monitor + self._closed = False + + @classmethod + def connect(cls) -> TeachCollectionSession: + """Attach to and validate the canonical teach collection modules.""" + client = Dimos.connect() + try: + modules = {info.instance_name: info for info in client.list_modules()} + for name, rpcs in { + _MONITOR: {"command", "get_status"}, + _COORDINATOR: {"list_tasks"}, + }.items(): + info = modules.get(name) + if info is None: + raise RuntimeError(f"running stack has no {name!r} module") + available = {rpc.name for rpc in info.rpcs} + missing = rpcs - available + if missing: + raise RuntimeError(f"{name!r} is missing RPCs: {sorted(missing)}") + + monitor = cast("Any", client.get_module(_MONITOR)) + coordinator = cast("Any", client.get_module(_COORDINATOR)) + tasks = set(coordinator.list_tasks()) + missing_tasks = _REQUIRED_TASKS - tasks + if missing_tasks: + raise RuntimeError(f"ControlCoordinator is missing tasks: {sorted(missing_tasks)}") + monitor.get_status() + return cls(client, monitor) + except Exception: + client.stop() + raise + + def get_status(self) -> EpisodeStatus: + """Read the monitor's latest state.""" + status = self._monitor.get_status() + if not isinstance(status, EpisodeStatus): + raise RuntimeError( + f"EpisodeMonitorModule returned {type(status).__name__}, expected EpisodeStatus" + ) + return status + + def command(self, event: str) -> EpisodeStatus: + """Send one episode command.""" + status = self._monitor.command(event) + if not isinstance(status, EpisodeStatus): + raise RuntimeError( + f"EpisodeMonitorModule returned {type(status).__name__}, expected EpisodeStatus" + ) + return status + + def close(self) -> None: + """Close only this RPC client; leave the daemon and robot running.""" + if not self._closed: + self._client.stop() + self._closed = True + + +class TeachCollectionApp(App[None]): + """Operator dashboard for hand-guided teach collection.""" + + CSS_PATH = theme.CSS_PATH + CSS = f""" + Screen {{ + align: center middle; + background: {theme.BACKGROUND}; + }} + + #dashboard {{ + width: 82; + max-width: 95%; + height: auto; + padding: 1 2; + border: double {theme.BORDER}; + background: {theme.BG}; + }} + + #title {{ + height: 1; + content-align: center middle; + color: {theme.ACCENT}; + text-style: bold; + }} + + #task {{ + height: 1; + text-align: center; + color: {theme.WHITE}; + }} + + #state {{ + height: 3; + margin-top: 1; + border: round {theme.SUCCESS}; + content-align: center middle; + color: {theme.SUCCESS}; + text-style: bold; + }} + + #state.recording, #state.disconnected {{ + border: round {theme.ERROR}; + color: {theme.ERROR}; + }} + + #counters {{ + height: 3; + }} + + .counter {{ + width: 1fr; + margin: 0 1; + border: round {theme.DIM}; + content-align: center middle; + text-align: center; + }} + + #guidance {{ + height: 3; + content-align: center middle; + text-align: center; + color: {theme.FOREGROUND}; + }} + + #message {{ + height: 2; + content-align: center middle; + text-align: center; + color: {theme.WARNING}; + }} + + #actions {{ + height: 3; + }} + + #actions Button {{ + width: 1fr; + margin: 0 1; + }} + """ + + BINDINGS = [ + Binding("space", "toggle_recording", "Start / save"), + Binding("d", "discard", "Discard"), + Binding("q", "quit", "Detach"), + Binding("ctrl+c", "quit", "Detach", show=False), + ] + + def __init__(self, session: TeachCollectionSession) -> None: + super().__init__() + self._session = session + self._status = session.get_status() + self._message = "Reset the scene, then start a take." + self._detached = False + self._recording_started_at: float | None = None + + def compose(self) -> ComposeResult: + with Container(id="dashboard"): + yield Static("OPENYAM / TEACH COLLECTION", id="title") + yield Static(id="task") + yield Static(id="state") + with Horizontal(id="counters"): + yield Static(id="saved", classes="counter") + yield Static(id="discarded", classes="counter") + yield Static(id="guidance") + yield Static(id="message") + with Horizontal(id="actions"): + yield Button("Start recording", id="toggle", variant="success") + yield Button("Discard", id="discard", variant="error", disabled=True) + yield Button("Detach", id="detach") + yield Footer() + + def on_mount(self) -> None: + self._refresh() + self.set_interval(0.25, self._poll) + + def on_unmount(self) -> None: + self._session.close() + + @staticmethod + def _format_elapsed(seconds: float) -> str: + minutes, seconds = divmod(max(seconds, 0.0), 60.0) + return f"{int(minutes):02d}:{seconds:04.1f}" + + def _set_status(self, status: EpisodeStatus) -> None: + was_recording = self._status.state == "recording" + self._status = status + recording = status.state == "recording" + if recording and not was_recording: + self._recording_started_at = time.monotonic() + elif not recording: + self._recording_started_at = None + + def _state_text(self) -> str: + if self._detached: + return "DISCONNECTED" + recording = self._status.state == "recording" + if not recording: + return "READY" + elapsed = ( + "--:--" + if self._recording_started_at is None + else self._format_elapsed(time.monotonic() - self._recording_started_at) + ) + return f"● RECORDING {elapsed}" + + def _refresh(self) -> None: + recording = self._status.state == "recording" + state = self.query_one("#state", Static) + state.set_class(recording and not self._detached, "recording") + state.set_class(self._detached, "disconnected") + state.update(self._state_text()) + + task = self._status.task_label or "Untitled task" + self.query_one("#task", Static).update(f"TASK {task}") + self.query_one("#saved", Static).update(f"SAVED\n{self._status.episodes_saved}") + self.query_one("#discarded", Static).update(f"DISCARDED\n{self._status.episodes_discarded}") + guidance = ( + "Move the gravity-compensated arm and passive gripper by hand.\n" + "Press Space to save this episode, or D to discard it." + if recording + else "Reset the scene and place the arm at the starting pose.\n" + "Press Space when the demonstration begins." + ) + if self._detached: + guidance = "The RPC connection closed. The daemon and arm are still running." + self.query_one("#guidance", Static).update(guidance) + self.query_one("#message", Static).update(self._message) + + toggle = self.query_one("#toggle", Button) + toggle.label = "Save episode" if recording else "Start recording" + toggle.variant = "error" if recording else "success" + toggle.disabled = self._detached + self.query_one("#discard", Button).disabled = self._detached or not recording + detach = self.query_one("#detach", Button) + detach.label = "Exit" if self._detached else "Detach" + detach.disabled = recording and not self._detached + + def _poll(self) -> None: + if self._detached: + return + try: + self._set_status(self._session.get_status()) + self._refresh() + except Exception as exc: + self._fail(exc) + + def _fail(self, exc: Exception) -> None: + self._message = f"Connection error: {exc}" + self._detached = True + self._session.close() + self._refresh() + + def _episode_command(self, event: str) -> None: + if self._detached: + return + try: + self._set_status(self._session.command(event)) + self._message = { + "start": "Recording. Drag the arm through the demonstration.", + "save": "Episode saved. Reset the scene for the next take.", + "discard": "Episode discarded. Reset the scene and try again.", + }.get(self._status.last_event, self._status.last_event) + self._refresh() + except Exception as exc: + self._fail(exc) + + def action_toggle_recording(self) -> None: + self._episode_command("toggle") + + def action_discard(self) -> None: + if self._status.state != "recording": + self._message = "Nothing to discard. Start a take first." + self._refresh() + return + self._episode_command("discard") + + def on_button_pressed(self, event: Button.Pressed) -> None: + actions = { + "toggle": self.action_toggle_recording, + "discard": self.action_discard, + "detach": self.action_quit, + } + action = actions.get(event.button.id or "") + if action is not None: + action() + + def action_quit(self) -> None: # type: ignore[override] + if not self._detached and self._status.state == "recording": + self._message = "Save with Space or discard with D before detaching." + self._refresh() + return + self.exit() + + +def collect() -> None: + """Control an already-running OpenYAM teach collection stack.""" + try: + session = TeachCollectionSession.connect() + except Exception as exc: + typer.echo(f"Unable to attach collection controls: {exc}", err=True) + raise typer.Exit(1) from exc + TeachCollectionApp(session).run() diff --git a/dimos/cli/commands/test_collect.py b/dimos/cli/commands/test_collect.py new file mode 100644 index 0000000000..14a99abcac --- /dev/null +++ b/dimos/cli/commands/test_collect.py @@ -0,0 +1,153 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from pytest_mock import MockerFixture +from textual.widgets import Button, Static + +from dimos.cli.commands.collect import TeachCollectionApp, TeachCollectionSession +from dimos.core.introspection.module.info import ModuleInfo, RpcInfo +from dimos.msgs.imitation_msgs.EpisodeStatus import EpisodeStatus + + +def _status( + state: str = "idle", + *, + event: str = "init", + saved: int = 0, + discarded: int = 0, +) -> EpisodeStatus: + return EpisodeStatus( + ts=1.0, + state=state, # type: ignore[arg-type] + episodes_saved=saved, + episodes_discarded=discarded, + last_event=event, # type: ignore[arg-type] + task_label="pick up the block", + ) + + +def _session(mocker: MockerFixture) -> tuple[TeachCollectionSession, Any, Any]: + client = mocker.Mock() + monitor = mocker.Mock() + monitor.get_status.return_value = _status() + monitor.command.side_effect = [ + _status("recording", event="start"), + _status("idle", event="discard", discarded=1), + ] + return TeachCollectionSession(client, monitor), client, monitor + + +def test_session_routes_episode_commands(mocker: MockerFixture) -> None: + session, _, monitor = _session(mocker) + + assert session.command("toggle").state == "recording" + + monitor.command.assert_called_once_with("toggle") + + +def test_panel_actions_route_keys_and_guard_quit(mocker: MockerFixture) -> None: + session, _, monitor = _session(mocker) + app = TeachCollectionApp(session) + mocker.patch.object(app, "_refresh") + exit_mock = mocker.patch.object(app, "exit") + + app.action_toggle_recording() + app.action_quit() + app.action_discard() + app.action_quit() + + assert monitor.command.call_args_list == [mocker.call("toggle"), mocker.call("discard")] + exit_mock.assert_called_once_with() + + +def test_panel_binds_the_documented_keys() -> None: + assert {binding.key: binding.action for binding in TeachCollectionApp.BINDINGS} == { + "space": "toggle_recording", + "d": "discard", + "q": "quit", + "ctrl+c": "quit", + } + + +async def test_dashboard_buttons_follow_episode_state(mocker: MockerFixture) -> None: + session, _, monitor = _session(mocker) + app = TeachCollectionApp(session) + mocker.patch.object(app, "set_interval") + + async with app.run_test(size=(80, 24)) as pilot: + assert str(app.query_one("#state", Static).render()) == "READY" + assert str(app.query_one("#toggle", Button).label) == "Start recording" + assert app.query_one("#discard", Button).disabled + + await pilot.click("#toggle") + + assert "RECORDING" in str(app.query_one("#state", Static).render()) + assert str(app.query_one("#toggle", Button).label) == "Save episode" + assert not app.query_one("#discard", Button).disabled + assert app.query_one("#detach", Button).disabled + + await pilot.click("#discard") + + assert str(app.query_one("#state", Static).render()) == "READY" + assert str(app.query_one("#discarded", Static).render()) == "DISCARDED\n1" + assert not app.query_one("#detach", Button).disabled + assert monitor.command.call_args_list == [mocker.call("toggle"), mocker.call("discard")] + + +def test_dashboard_formats_recording_time() -> None: + assert TeachCollectionApp._format_elapsed(0.0) == "00:00.0" + assert TeachCollectionApp._format_elapsed(62.34) == "01:02.3" + + +async def test_rpc_failure_disables_controls_without_stopping_daemon( + mocker: MockerFixture, +) -> None: + session, client, monitor = _session(mocker) + app = TeachCollectionApp(session) + mocker.patch.object(app, "set_interval") + monitor.command.side_effect = RuntimeError("stack disappeared") + + async with app.run_test(size=(80, 24)) as pilot: + await pilot.click("#toggle") + + assert str(app.query_one("#state", Static).render()) == "DISCONNECTED" + assert app.query_one("#toggle", Button).disabled + assert app.query_one("#discard", Button).disabled + assert not app.query_one("#detach", Button).disabled + assert str(app.query_one("#detach", Button).label) == "Exit" + + # Closing the failed dashboard must only detach this RPC client. + client.stop.assert_called_once_with() + + +def test_connect_rejects_the_wrong_stack_and_closes_client(mocker: MockerFixture) -> None: + client = mocker.Mock() + client.list_modules.return_value = [ + ModuleInfo( + name="EpisodeMonitorModule", + instance_name="EpisodeMonitorModule", + rpcs=[RpcInfo(name="command"), RpcInfo(name="get_status")], + ) + ] + mocker.patch("dimos.cli.commands.collect.Dimos.connect", return_value=client) + + try: + TeachCollectionSession.connect() + except RuntimeError as exc: + assert "ControlCoordinator" in str(exc) + else: + raise AssertionError("wrong stack should fail validation") + client.stop.assert_called_once_with() diff --git a/dimos/cli/dimos.py b/dimos/cli/dimos.py index 3bc1df6be9..08d8a74be5 100644 --- a/dimos/cli/dimos.py +++ b/dimos/cli/dimos.py @@ -52,6 +52,7 @@ from dimos.cli.commands.apriltag import apriltag from dimos.cli.commands.bake import bake from dimos.cli.commands.cameracalibrate import cameracalibrate +from dimos.cli.commands.collect import collect from dimos.cli.commands.data import data_app from dimos.cli.commands.dataprep import dataprep_app from dimos.cli.commands.docs import docs @@ -128,6 +129,7 @@ def cli_main() -> None: )(bake) main.command(name="list")(list_blueprints) main.command()(docs) +main.command()(collect) main.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})(spy) main.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})(lcmspy) main.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})(agentspy) diff --git a/dimos/experimental/memory/README.md b/dimos/experimental/memory/README.md index 0537a95ce4..ca07570ac1 100644 --- a/dimos/experimental/memory/README.md +++ b/dimos/experimental/memory/README.md @@ -11,8 +11,9 @@ The recorder is built as a locked Nix package. Nix supplies Rust, CMake, NASM, SQLite, and the native libraries used by TurboJPEG, so none of those tools or development packages need to be installed on the host. -The Python module builds the package automatically on first use. To build it -ahead of time, run: +The Python module resolves the package through Nix before each launch, so its +wire protocol always matches the Python checkout. Nix reuses the cached package +when the native sources have not changed. To build it ahead of time, run: ```bash cd dimos/experimental/memory/rust diff --git a/dimos/experimental/memory/rust/flake.nix b/dimos/experimental/memory/rust/flake.nix index 0a950fafe4..f9057c2a25 100644 --- a/dimos/experimental/memory/rust/flake.nix +++ b/dimos/experimental/memory/rust/flake.nix @@ -38,7 +38,7 @@ cargoLock = { lockFile = ../../../../Cargo.lock; outputHashes = { - "dimos-lcm-0.1.0" = "sha256-GGkx4Mn6NYP6KZecmoRLKGWIih/+y8OgNn12DeXX6n8="; + "dimos-lcm-0.1.0" = "sha256-Z0tKEjNb/VIyfJcYTeZjWhWWYMhQ8wtoe8TjhonPUns="; }; }; diff --git a/dimos/hardware/whole_body/damiao/adapter.py b/dimos/hardware/whole_body/damiao/adapter.py index 75b1c75c33..ece418cc3f 100644 --- a/dimos/hardware/whole_body/damiao/adapter.py +++ b/dimos/hardware/whole_body/damiao/adapter.py @@ -78,6 +78,11 @@ def __init__( unknown_buses = config.bus_devices.keys() - set(self.bus_names) if unknown_buses: raise ValueError(f"unknown CAN bus overrides: {sorted(unknown_buses)}") + unknown_grippers = set(config.passive_grippers) - set(self.gripper_joints) + if unknown_grippers: + raise ValueError(f"unknown passive grippers: {sorted(unknown_grippers)}") + if len(config.passive_grippers) != len(set(config.passive_grippers)): + raise ValueError("passive_grippers contains duplicate names") if len(self.bus_names) != len(set(self.bus_names)): raise ValueError("Damiao topology contains duplicate logical bus names") @@ -240,6 +245,13 @@ def activate(self) -> bool: for arm in self._arms.values(): arm.set_mode("mit") self._robot.enable() + for name in self._runtime_config.passive_grippers: + gripper = self._grippers[name] + gripper.set_mode("mit") + gripper.enable() + gripper.mit_control(0.0, 0.0, float(gripper.motor.position), 0.0, 0.0) + if self._runtime_config.passive_grippers: + self._robot.tick(self._runtime_config.tick_deadline_us) self._active = True self.read_motor_states() return True @@ -335,6 +347,8 @@ def write_motor_commands(self, commands: list[MotorCommand]) -> bool: commands[arm_count:], strict=True, ): + if name in self._runtime_config.passive_grippers: + continue if not np.isfinite(command.q) or not 0.0 <= command.q <= 1.0: raise ValueError(f"gripper {name!r} opening must be in [0, 1]") @@ -363,7 +377,11 @@ def write_motor_commands(self, commands: list[MotorCommand]) -> bool: for name in self.gripper_joints: opening = commands[offset].q - self._grippers[name].set_opening(opening) + if name in self._runtime_config.passive_grippers: + gripper = self._grippers[name] + gripper.mit_control(0.0, 0.0, float(gripper.motor.position), 0.0, 0.0) + else: + self._grippers[name].set_opening(opening) offset += 1 self._robot.tick(self._runtime_config.tick_deadline_us) diff --git a/dimos/hardware/whole_body/damiao/config.py b/dimos/hardware/whole_body/damiao/config.py index d9b95092e2..ce0c3829bf 100644 --- a/dimos/hardware/whole_body/damiao/config.py +++ b/dimos/hardware/whole_body/damiao/config.py @@ -31,4 +31,5 @@ class DamiaoRuntimeConfig: bus_devices: dict[_NonEmptyString, _NonEmptyString] = Field(default_factory=dict) gravity_comp: bool = Field(default=True, strict=True) + passive_grippers: tuple[_NonEmptyString, ...] = () tick_deadline_us: int = Field(default=1_000, ge=1, strict=True) diff --git a/dimos/hardware/whole_body/damiao/test_adapter.py b/dimos/hardware/whole_body/damiao/test_adapter.py index c1739d4809..dd17c4d406 100644 --- a/dimos/hardware/whole_body/damiao/test_adapter.py +++ b/dimos/hardware/whole_body/damiao/test_adapter.py @@ -65,14 +65,31 @@ def mit_control(self, commands: np.ndarray) -> None: class FakeGripper: def __init__(self, opening: float) -> None: self.opening = opening + self.motor = Mock(position=opening) self.command_error: Exception | None = None self.commands: list[float] = [] + self.modes: list[str] = [] + self.mit_commands: list[tuple[float, float, float, float, float]] = [] + self.enable_count = 0 + self.disable_count = 0 def set_opening(self, opening: float) -> None: if self.command_error is not None: raise self.command_error self.commands.append(opening) + def disable(self) -> None: + self.disable_count += 1 + + def enable(self) -> None: + self.enable_count += 1 + + def set_mode(self, mode: str) -> None: + self.modes.append(mode) + + def mit_control(self, kp: float, kd: float, q: float, dq: float, tau: float) -> None: + self.mit_commands.append((kp, kd, q, dq, tau)) + class FakeTransport: def __init__(self) -> None: @@ -296,6 +313,14 @@ def test_init_unknown_bus_override_raises_value_error(dual_robot: FakeRobot) -> ) +def test_init_unknown_passive_gripper_raises_value_error(dual_robot: FakeRobot) -> None: + with pytest.raises(ValueError, match="unknown passive grippers"): + DualAdapter( + dual_robot, + runtime_config=DamiaoRuntimeConfig(passive_grippers=("missing",)), + ) + + def test_init_duplicate_logical_bus_names_raises_value_error(dual_robot: FakeRobot) -> None: class DuplicateBusAdapter(DualAdapter): bus_names = ("left", "left") @@ -360,12 +385,14 @@ def test_init_rehydrates_serialized_runtime_config(dual_robot: FakeRobot) -> Non runtime_config={ "bus_devices": {"left": "can8"}, "gravity_comp": False, + "passive_grippers": ["left_gripper"], "tick_deadline_us": 2_000, }, ) assert adapter._runtime_config.bus_devices == {"left": "can8"} assert adapter._runtime_config.gravity_comp is False + assert adapter._runtime_config.passive_grippers == ("left_gripper",) assert adapter._runtime_config.tick_deadline_us == 2_000 @@ -570,6 +597,32 @@ def test_activate_enable_failure_disables_robot( assert dual_robot.disable_count == 1 +def test_activate_enables_zero_impedance_for_configured_passive_gripper( + dual_robot: FakeRobot, + adapter_factory: Callable[..., DualAdapter], +) -> None: + adapter = adapter_factory( + dual_robot, + runtime_config=DamiaoRuntimeConfig( + gravity_comp=False, + passive_grippers=("left_gripper",), + ), + ) + assert adapter.connect() + + assert adapter.activate() + + left_gripper = cast("FakeGripper", dual_robot["left_gripper"]) + right_gripper = cast("FakeGripper", dual_robot["right_gripper"]) + assert left_gripper.disable_count == 0 + assert left_gripper.modes == ["mit"] + assert left_gripper.enable_count == 1 + assert left_gripper.mit_commands == [(0.0, 0.0, 0.5, 0.0, 0.0)] + assert right_gripper.modes == [] + assert right_gripper.enable_count == 0 + assert right_gripper.mit_commands == [] + + def test_deactivate_connected_adapter_disables_robot( active_dual_adapter: DualAdapter, dual_robot: FakeRobot, @@ -756,6 +809,32 @@ def test_write_motor_commands_grippers_routes_normalized_openings( assert cast("FakeGripper", dual_robot["right_gripper"]).commands == [0.75] +def test_write_motor_commands_keeps_passive_gripper_at_zero_impedance( + dual_robot: FakeRobot, + adapter_factory: Callable[..., DualAdapter], +) -> None: + adapter = adapter_factory( + dual_robot, + runtime_config=DamiaoRuntimeConfig( + gravity_comp=False, + passive_grippers=("left_gripper",), + ), + ) + assert adapter.connect() + assert adapter.activate() + commands = [MotorCommand(q=0.0)] * 4 + [MotorCommand(q=2.0), MotorCommand(q=0.75)] + + assert adapter.write_motor_commands(commands) + + left_gripper = cast("FakeGripper", dual_robot["left_gripper"]) + assert left_gripper.commands == [] + assert left_gripper.mit_commands == [ + (0.0, 0.0, 0.5, 0.0, 0.0), + (0.0, 0.0, 0.5, 0.0, 0.0), + ] + assert cast("FakeGripper", dual_robot["right_gripper"]).commands == [0.75] + + def test_write_motor_commands_combined_command_ticks_once( active_dual_adapter: DualAdapter, dual_robot: FakeRobot, diff --git a/dimos/imitation/README.md b/dimos/imitation/README.md index 846e5c3958..1adc870aef 100644 --- a/dimos/imitation/README.md +++ b/dimos/imitation/README.md @@ -1,11 +1,14 @@ # Imitation Learning Collect demonstrations, build training datasets, and run trained policies in -DimOS. Teleoperation records episodes to a SQLite or MCAP artifact, and DataPrep -converts that recording into a LeRobot or HDF5 dataset for imitation learning. +DimOS. Quest teleoperation or direct arm teaching records episodes to a SQLite +or MCAP artifact. DataPrep converts that recording into a LeRobot or HDF5 +dataset for imitation learning. ``` -teleop (Quest) ─▶ recorder ─▶ session__.db/.mcap ─▶ dimos dataprep ─▶ dataset +Quest teleop ─┐ + ├─▶ recorder ─▶ session__.db/.mcap ─▶ dimos dataprep ─▶ dataset +direct teach ─┘ ``` After training, use the production @@ -56,6 +59,99 @@ prints one line per transition: > End each good take with **B** before quitting — an episode still recording at > shutdown is dropped. +### OpenYAM direct teaching + +Direct teaching removes the Quest teleoperator. The arm runs with gravity +compensation, zero position stiffness, and joint damping. Move it by hand while +the existing OpenYAM observation and action streams are recorded. The coordinator's +idle trajectory holder keeps sending the motor commands that apply gravity and +damping; zero stiffness makes its latched position target inert. + +Start the hardware stack in one terminal. Be ready to support the arm as it +activates, and keep people and obstacles outside its workspace. + +```bash +dimos --can-port follower_l run learning-collect-teach-openyam --daemon \ + --task "pick up the red block" \ + --WristCamera.hardware.camera-index 0 \ + --nativecollectionrecorder.store.path data/recordings/openyam-teach.mcap +``` + +Attach the collection panel from another terminal: + +```bash +dimos collect +``` + +```text +╔════════════════ OPENYAM / TEACH COLLECTION ════════════════╗ +║ TASK pick up the red block ║ +║ ║ +║ ● RECORDING 00:08.4 ║ +║ ║ +║ SAVED 2 DISCARDED 0 ║ +║ ║ +║ Move the gravity-compensated arm and passive gripper. ║ +║ ║ +║ [ Save episode ] [ Discard ] [ Detach ] ║ +╚════════════════════════════════════════════════════════════╝ +``` + +| Key | Action | +| --- | --- | +| **Space** | Start an episode; press again to save it | +| **D** | Discard the in-progress episode | +| **Q** or **Ctrl-C** | Detach the panel while idle | + +The same actions are available as clickable buttons. +The panel refuses to detach while recording. Save or discard the take first. +Detaching closes only the panel's RPC connection; the arm remains active in +gravity-compensation mode until you run `dimos stop`. + +A complete session is: + +```text +start daemon ─▶ attach panel ─▶ start/save takes ─▶ detach panel ─▶ stop daemon +``` + +After collection, stop the stack cleanly and build the dataset with the +direct-teach profile: + +```bash +dimos stop +dimos dataprep build \ + --source data/recordings/openyam-teach.mcap \ + --profile dimos.robot.manipulators.openyam.learning:OPENYAM_TEACH_LEARNING_PROFILE \ + --output data/datasets/openyam-teach +``` + +Visualize an episode from the isolated LeRobot environment so its matching +Rerun viewer executable is available on `PATH`: + +```bash +uv run --project dimos/imitation/policy/lerobot/python --frozen \ + lerobot-dataset-viz \ + --repo-id local/openyam-wrist \ + --root "$PWD/data/datasets/openyam-teach" \ + --episode-index 0 \ + --mode local +``` + +The action row contains the measured arm and gripper positions at that instant. +The direct-teach profile therefore reads both state and action from the continuous +coordinator joint-state stream; Quest collection continues to use accepted commands. + +Before a production collection, run one hardware smoke test: + +1. Support the arm, start the daemon, and confirm that the arm and gripper can + be moved by hand without position-hold resistance. The arm should retain + light joint damping while the enabled gripper runs with zero stiffness, + zero damping, and zero feed-forward torque. +2. Attach `dimos collect` and verify that the panel reports the gripper as + passive. +3. Record and save a short take, stop the daemon, then inspect the MCAP with + `dimos dataprep inspect` and `OPENYAM_TEACH_LEARNING_PROFILE`. + ### Where the recording goes ``` diff --git a/dimos/imitation/collection/episode_monitor.py b/dimos/imitation/collection/episode_monitor.py index bc0150efd0..792fe55bff 100644 --- a/dimos/imitation/collection/episode_monitor.py +++ b/dimos/imitation/collection/episode_monitor.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Single point of Quest-input → EpisodeStatus translation. +"""Single point of operator-input → EpisodeStatus translation. -Watches buttons, runs the start/save/discard state machine, +Watches Quest buttons and accepts RPC commands, runs the episode state machine, publishes EpisodeStatus on every transition. RecordReplay (or whatever records the bus) captures that stream into session.db; DataPrep reads only -the recorded EpisodeStatus events offline — never raw buttons. +the recorded EpisodeStatus events offline — never raw operator input. """ from __future__ import annotations @@ -93,6 +93,7 @@ def __init__(self, **kwargs: Any) -> None: self._state: RecordingState = "idle" self._saved: int = 0 self._discarded: int = 0 + self._last_event: EpisodeEvent = "init" self._lock = threading.Lock() self._transition_lock = threading.Lock() self._stopping = False @@ -122,6 +123,7 @@ def stop(self) -> None: if self._state == "recording": self._discarded += 1 self._state = "idle" + self._last_event = "discard" status = self._snapshot("discard", time.time()) else: status = None @@ -148,17 +150,30 @@ def _on_buttons(self, msg: Buttons) -> None: for event_name in fired: self._transition(event_name, ts) - def _transition(self, event: EpisodeCommand, ts: float) -> None: + @rpc + def command(self, event: EpisodeCommand) -> EpisodeStatus: + """Apply an episode command from an attached operator interface.""" + return self._transition(event, time.time()) + + @rpc + def get_status(self) -> EpisodeStatus: + """Return the latest episode state without publishing a new event.""" + with self._lock: + return self._snapshot(self._last_event, time.time()) + + def _transition(self, event: EpisodeCommand, ts: float) -> EpisodeStatus: """State-machine transition. Publishes EpisodeStatus on every change. ``toggle`` resolves to ``start`` when idle and ``save`` when recording, so one button can begin and end a take. The resolved event is what gets published (DataPrep only ever sees start/save/discard). """ + if event not in ("start", "save", "discard", "toggle"): + raise ValueError(f"unknown episode command: {event!r}") with self._transition_lock: with self._lock: if self._stopping: - return + return self._snapshot(self._last_event, ts) if event == "toggle": event = "save" if self._state == "recording" else "start" if event == "start": @@ -174,9 +189,10 @@ def _transition(self, event: EpisodeCommand, ts: float) -> None: if self._state == "recording": self._discarded += 1 self._state = "idle" + self._last_event = event # Snapshot under the mutation's lock so the event matches the state. status = self._snapshot(event, ts) - self._emit(status) + return self._emit(status) def _snapshot(self, last_event: EpisodeEvent, ts: float) -> EpisodeStatus: """Build a status from current state. Caller must hold `self._lock`.""" diff --git a/dimos/imitation/collection/test_episode_monitor.py b/dimos/imitation/collection/test_episode_monitor.py index 93a454abaf..bbfb416576 100644 --- a/dimos/imitation/collection/test_episode_monitor.py +++ b/dimos/imitation/collection/test_episode_monitor.py @@ -93,6 +93,46 @@ def test_toggle_starts_then_saves(make_monitor: Callable[..., EpisodeMonitorModu assert events[-1].task_label == "pick up the block" +def test_rpc_commands_use_the_same_state_machine( + make_monitor: Callable[..., EpisodeMonitorModule], +) -> None: + m = make_monitor() + + recording = m.command("toggle") + saved = m.command("toggle") + + assert [event.last_event for event in _events(m)] == ["start", "save"] + assert recording.state == "recording" + assert saved.state == "idle" + assert saved.episodes_saved == 1 + + +def test_get_status_does_not_publish( + make_monitor: Callable[..., EpisodeMonitorModule], +) -> None: + m = make_monitor() + m.command("start") + event_count = len(_events(m)) + + status = m.get_status() + + assert status.state == "recording" + assert status.last_event == "start" + assert len(_events(m)) == event_count + + +def test_invalid_rpc_command_is_rejected_without_changing_state( + make_monitor: Callable[..., EpisodeMonitorModule], +) -> None: + m = make_monitor() + + with pytest.raises(ValueError, match="unknown episode command"): + m.command("pause") # type: ignore[arg-type] + + assert m.get_status().state == "idle" + assert _events(m) == [] + + def test_task_is_required(make_monitor: Callable[..., EpisodeMonitorModule]) -> None: with pytest.raises(ValidationError, match="task"): EpisodeMonitorModule() diff --git a/dimos/imitation/dataprep/lerobot.py b/dimos/imitation/dataprep/lerobot.py index eb7de9218f..b6fc404330 100644 --- a/dimos/imitation/dataprep/lerobot.py +++ b/dimos/imitation/dataprep/lerobot.py @@ -72,7 +72,7 @@ def _run(request: Request) -> Result: output = result.stderr.strip() or result.stdout.strip() raise RuntimeError(f"LeRobot dataprep exited with status {result.returncode}: {output}") try: - return RESULT_ADAPTER.validate_json(result.stdout) + return RESULT_ADAPTER.validate_json(result.stdout.rstrip().rsplit("\n", 1)[-1]) except ValueError as error: raise RuntimeError( f"LeRobot dataprep returned an invalid result: {result.stdout!r}" @@ -81,6 +81,12 @@ def _run(request: Request) -> Result: def run_lerobot_dataprep(config: DataPrepConfig) -> Path: """Build a dataset in the isolated LeRobot environment.""" + config = config.model_copy( + update={ + "source": str(Path(config.source).resolve()), + "output": config.output.model_copy(update={"path": config.output.path.resolve()}), + } + ) result = _run(BuildRequest(config=config)) if not isinstance(result, BuildResult): raise RuntimeError(f"LeRobot dataprep returned {result.command!r} for a build request") @@ -89,7 +95,7 @@ def run_lerobot_dataprep(config: DataPrepConfig) -> Path: def inspect_lerobot_dataset(path: Path) -> dict[str, Any]: """Inspect a dataset in the isolated LeRobot environment.""" - result = _run(InspectRequest(path=path)) + result = _run(InspectRequest(path=path.resolve())) if not isinstance(result, InspectResult): raise RuntimeError(f"LeRobot dataprep returned {result.command!r} for an inspect request") return result.info diff --git a/dimos/imitation/dataprep/test_lerobot_cli.py b/dimos/imitation/dataprep/test_lerobot_cli.py index 5ea982c112..b55c51eb7b 100644 --- a/dimos/imitation/dataprep/test_lerobot_cli.py +++ b/dimos/imitation/dataprep/test_lerobot_cli.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json from pathlib import Path import subprocess @@ -92,7 +93,12 @@ def test_conversion_runs_packaged_module_in_policy_project( run = mocker.patch( "dimos.imitation.dataprep.lerobot.subprocess.run", return_value=subprocess.CompletedProcess( - [], 0, stdout=f'{{"command":"build","path":"{tmp_path / "dataset"}"}}', stderr="" + [], + 0, + stdout=( + f'[dataprep] wrote 1 episode\n{{"command":"build","path":"{tmp_path / "dataset"}"}}' + ), + stderr="", ), ) config = DataPrepConfig( @@ -113,6 +119,9 @@ def test_conversion_runs_packaged_module_in_policy_project( assert run.call_args.kwargs["capture_output"] is True assert run.call_args.kwargs["text"] is True assert '"command":"build"' in run.call_args.kwargs["input"] + assert json.loads(run.call_args.kwargs["input"])["config"]["source"] == str( + Path("recording.db").resolve() + ) def test_conversion_reports_missing_uv(tmp_path: Path, mocker: pytest_mock.MockerFixture) -> None: @@ -148,8 +157,9 @@ def test_conversion_reports_child_process_diagnostics( def test_inspection_uses_the_same_isolated_entrypoint( - tmp_path: Path, mocker: pytest_mock.MockerFixture + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mocker: pytest_mock.MockerFixture ) -> None: + monkeypatch.chdir(tmp_path) run = mocker.patch( "dimos.imitation.dataprep.lerobot.subprocess.run", return_value=subprocess.CompletedProcess( @@ -157,5 +167,6 @@ def test_inspection_uses_the_same_isolated_entrypoint( ), ) - assert inspect_lerobot_dataset(tmp_path / "dataset") == {"format": "lerobot"} + assert inspect_lerobot_dataset(Path("dataset")) == {"format": "lerobot"} assert '"command":"inspect"' in run.call_args.kwargs["input"] + assert json.loads(run.call_args.kwargs["input"])["path"] == str(tmp_path / "dataset") diff --git a/dimos/imitation/policy/lerobot/python/dimos_lerobot/mcap_dataprep_tests.py b/dimos/imitation/policy/lerobot/python/dimos_lerobot/mcap_dataprep_tests.py index 2bd7bdc6bf..3cc166e262 100644 --- a/dimos/imitation/policy/lerobot/python/dimos_lerobot/mcap_dataprep_tests.py +++ b/dimos/imitation/policy/lerobot/python/dimos_lerobot/mcap_dataprep_tests.py @@ -21,6 +21,7 @@ from typing import Any from dimos_lerobot.dataprep import write +from lerobot.datasets.lerobot_dataset import LeRobotDataset from mcap.writer import Writer as McapWriter import numpy as np @@ -197,3 +198,7 @@ def test_mcap_converts_to_lerobot_dataset(tmp_path: Path) -> None: assert info["total_frames"] == 3 assert info["fps"] == 30 assert info["features"]["action"]["names"] == JOINTS + + dataset = LeRobotDataset("local/openyam-mcap", root=root) + frame = dataset[0] + assert tuple(frame["observation.images.wrist"].shape) == (3, 64, 64) diff --git a/dimos/imitation/policy/lerobot/python/pyproject.toml b/dimos/imitation/policy/lerobot/python/pyproject.toml index 27fcf8516c..83c9c7a055 100644 --- a/dimos/imitation/policy/lerobot/python/pyproject.toml +++ b/dimos/imitation/policy/lerobot/python/pyproject.toml @@ -26,6 +26,9 @@ override-dependencies = [ # LeRobot's headless OpenCV wheel owns the same cv2/ tree as DimOS's # opencv-contrib-python dependency. Contrib is the required superset. "opencv-python-headless; sys_platform == 'never'", + # LeRobot selects TorchCodec whenever it is importable, but its native wheel + # cannot load against newer system FFmpeg ABIs. Use LeRobot's PyAV backend. + "torchcodec; sys_platform == 'never'", ] [tool.setuptools.packages.find] diff --git a/dimos/imitation/policy/lerobot/python/uv.lock b/dimos/imitation/policy/lerobot/python/uv.lock index f6edf74a0a..66fe2d4b74 100644 --- a/dimos/imitation/policy/lerobot/python/uv.lock +++ b/dimos/imitation/policy/lerobot/python/uv.lock @@ -10,7 +10,10 @@ resolution-markers = [ ] [manifest] -overrides = [{ name = "opencv-python-headless", marker = "sys_platform == 'never'" }] +overrides = [ + { name = "opencv-python-headless", marker = "sys_platform == 'never'" }, + { name = "torchcodec", marker = "sys_platform == 'never'" }, +] [[package]] name = "accelerate" @@ -636,7 +639,7 @@ dataset = [ { name = "jsonlines" }, { name = "pandas" }, { name = "pyarrow" }, - { name = "torchcodec", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'AMD64' and sys_platform == 'linux') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'arm64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "torchcodec", marker = "sys_platform == 'never'" }, ] viz = [ { name = "foxglove-sdk" }, @@ -1447,12 +1450,6 @@ wheels = [ name = "torchcodec" version = "0.11.1" source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/85/38f4843ff2a6bf7dfb71a153acd99024dadb96749965a67524c2f1cc1894/torchcodec-0.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:57056e91d1d883d0fb77ca7759e304be9c0bdb4ea0e37bde5c2e361347063b8c", size = 4368988, upload-time = "2026-04-14T18:24:51.46Z" }, - { url = "https://files.pythonhosted.org/packages/4b/85/3b41034b0f1289423745f918ace2a1e1e86b9c578c2e2461b6afcbb5354a/torchcodec-0.11.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f1aee486a84247fcaa67870ac5005aa8d382a9839e91e476fa71b5b3d9fda9b7", size = 2397532, upload-time = "2026-04-14T18:24:53.368Z" }, - { url = "https://files.pythonhosted.org/packages/ca/a9/a2b6ee3e84c55bdd0c45fd991dde71c95a99115ec9e26938b212b4545dcf/torchcodec-0.11.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6c26e90e7aa982302644d0af8cb706318682bb390f48a80ecbfeab03499acd04", size = 2329883, upload-time = "2026-04-14T18:24:55.467Z" }, - { url = "https://files.pythonhosted.org/packages/82/48/683114a4ed6b59f76b6919532a5db0f4068787be26bab92cc18a1dfa6794/torchcodec-0.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:3fd2d10e0e0a5f455c1c87dc1380b3bd43b77dd5eeeaf479470643b1c04a2dd2", size = 1921066, upload-time = "2026-04-14T18:24:57.102Z" }, -] [[package]] name = "torchvision" diff --git a/dimos/mapping/ray_tracing/rust/Cargo.toml b/dimos/mapping/ray_tracing/rust/Cargo.toml index 501f8c5028..73445f8093 100644 --- a/dimos/mapping/ray_tracing/rust/Cargo.toml +++ b/dimos/mapping/ray_tracing/rust/Cargo.toml @@ -33,7 +33,7 @@ region_bounds = "geometry_msgs.PoseStamped" [dependencies] dimos-module = { path = "../../../../native/rust/dimos-module" } -lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" } +lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", rev = "dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5" } tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal"] } serde = { version = "1", features = ["derive"] } ahash = "0.8" diff --git a/dimos/navigation/nav_3d/mls_planner/rust/Cargo.toml b/dimos/navigation/nav_3d/mls_planner/rust/Cargo.toml index 2c5d39de04..b6e1a82fa2 100644 --- a/dimos/navigation/nav_3d/mls_planner/rust/Cargo.toml +++ b/dimos/navigation/nav_3d/mls_planner/rust/Cargo.toml @@ -36,7 +36,7 @@ path = "nav_msgs.Path" [dependencies] dimos-module = { path = "../../../../../native/rust/dimos-module" } -lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" } +lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", rev = "dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5" } tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal"] } serde = { version = "1", features = ["derive"] } ahash = "0.8" diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 753fbf5274..4dd573d5b8 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -78,6 +78,7 @@ "learning-collect-quest-openyam": "dimos.robot.manipulators.openyam.blueprints.learning_collection:learning_collect_quest_openyam", "learning-collect-quest-piper": "dimos.imitation.collection.blueprint:learning_collect_quest_piper", "learning-collect-quest-xarm7": "dimos.imitation.collection.blueprint:learning_collect_quest_xarm7", + "learning-collect-teach-openyam": "dimos.robot.manipulators.openyam.blueprints.learning_collection:learning_collect_teach_openyam", "learning-rollout-quest-openyam": "dimos.robot.manipulators.openyam.blueprints.learning_rollout:learning_rollout_quest_openyam", "mid360": "dimos.hardware.sensors.lidar.livox.livox_blueprints:mid360", "mid360-fastlio": "dimos.hardware.sensors.lidar.fastlio2.fastlio_blueprints:mid360_fastlio", diff --git a/dimos/robot/manipulators/openyam/blueprints/learning_collection.py b/dimos/robot/manipulators/openyam/blueprints/learning_collection.py index a8d3b1fe07..784eaaf71d 100644 --- a/dimos/robot/manipulators/openyam/blueprints/learning_collection.py +++ b/dimos/robot/manipulators/openyam/blueprints/learning_collection.py @@ -16,16 +16,24 @@ from __future__ import annotations +from dataclasses import replace from datetime import datetime from dimos.constants import STATE_DIR -from dimos.core.coordination.blueprints import autoconnect +from dimos.control.coordinator import ControlCoordinator, TaskConfig +from dimos.core.coordination.blueprints import Blueprint, autoconnect from dimos.experimental.memory.rust_recorder import RustMcapStoreConfig from dimos.hardware.sensors.camera.module import CameraModule from dimos.hardware.sensors.camera.webcam import WebcamConfig +from dimos.hardware.whole_body.damiao.config import DamiaoRuntimeConfig +from dimos.hardware.whole_body.spec import WholeBodyConfig from dimos.imitation.collection.episode_monitor import EpisodeMonitorModule from dimos.imitation.collection.native_recorder import NativeCollectionRecorder from dimos.robot.manipulators.openyam.blueprints.teleop import teleop_quest_openyam +from dimos.robot.manipulators.openyam.config import ( + OPENYAM_JOINTS, + openyam_hardware, +) from dimos.robot.manipulators.openyam.learning import OPENYAM_LEARNING_PROFILE @@ -33,14 +41,14 @@ def _session_mcap() -> str: return str(STATE_DIR / "recordings" / f"session_openyam_{datetime.now():%Y%m%d_%H%M%S}.mcap") -learning_collect_quest_openyam = autoconnect( - NativeCollectionRecorder.blueprint( - store=RustMcapStoreConfig(path=_session_mcap()), - record_tf=False, - ), - EpisodeMonitorModule.blueprint(), - teleop_quest_openyam, - CameraModule.blueprint( +def _teach_session_mcap() -> str: + return str( + STATE_DIR / "recordings" / f"session_openyam_teach_{datetime.now():%Y%m%d_%H%M%S}.mcap" + ) + + +def _wrist_camera() -> Blueprint: + return CameraModule.blueprint( instance_name="WristCamera", hardware=WebcamConfig( camera_index=0, @@ -50,5 +58,59 @@ def _session_mcap() -> str: frame_id_prefix=OPENYAM_LEARNING_PROFILE.camera_frame_prefix, ), frame_id=OPENYAM_LEARNING_PROFILE.camera_frame_id, + ) + + +learning_collect_quest_openyam = autoconnect( + NativeCollectionRecorder.blueprint( + store=RustMcapStoreConfig(path=_session_mcap()), + record_tf=False, + ), + EpisodeMonitorModule.blueprint(), + teleop_quest_openyam, + _wrist_camera(), +) + + +OPENYAM_TEACH_DAMPING = (2.0, 2.0, 2.0, 0.5, 0.5, 0.5, 0.0) +_openyam_teach_hardware = openyam_hardware() +if _openyam_teach_hardware.adapter_type == "openyam_damiao": + runtime_config = _openyam_teach_hardware.adapter_kwargs["runtime_config"] + if not isinstance(runtime_config, DamiaoRuntimeConfig): + raise TypeError("OpenYAM Damiao hardware requires DamiaoRuntimeConfig") + _openyam_teach_hardware = replace( + _openyam_teach_hardware, + adapter_kwargs={ + **_openyam_teach_hardware.adapter_kwargs, + "runtime_config": replace(runtime_config, passive_grippers=("gripper",)), + }, + ) +_openyam_teach_hardware = replace( + _openyam_teach_hardware, + wb_config=WholeBodyConfig( + kp=(0.0,) * len(OPENYAM_JOINTS), + kd=OPENYAM_TEACH_DAMPING, + ), +) + +learning_collect_teach_openyam = autoconnect( + NativeCollectionRecorder.blueprint( + store=RustMcapStoreConfig(path=_teach_session_mcap()), + record_tf=False, + ), + EpisodeMonitorModule.blueprint(), + ControlCoordinator.blueprint( + instance_name="ControlCoordinator", + hardware=[_openyam_teach_hardware], + tasks=[ + TaskConfig( + name="teach_openyam", + type="trajectory", + joint_names=list(OPENYAM_JOINTS), + priority=10, + params={"hold_position_when_idle": True}, + ), + ], ), + _wrist_camera(), ) diff --git a/dimos/robot/manipulators/openyam/blueprints/test_learning_collection.py b/dimos/robot/manipulators/openyam/blueprints/test_learning_collection.py index 43270c3116..cb17605355 100644 --- a/dimos/robot/manipulators/openyam/blueprints/test_learning_collection.py +++ b/dimos/robot/manipulators/openyam/blueprints/test_learning_collection.py @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from dimos.control.coordinator import ControlCoordinator from dimos.core.coordination.blueprint_config.parser import BlueprintConfigParser +from dimos.hardware.sensors.camera.module import CameraModule +from dimos.imitation.collection.episode_monitor import EpisodeMonitorModule from dimos.imitation.collection.native_recorder import NativeCollectionRecorder from dimos.robot.manipulators.openyam.blueprints.learning_collection import ( learning_collect_quest_openyam, + learning_collect_teach_openyam, ) +from dimos.robot.manipulators.openyam.config import OPENYAM_JOINTS def test_openyam_collection_uses_the_native_recorder() -> None: @@ -50,3 +55,65 @@ def test_native_openyam_paths_are_configurable_from_cli() -> None: ) assert parsed.module_kwargs("episodemonitormodule")["task"] == "pick up the red block" assert learning_collect_quest_openyam.active_blueprints[0].kwargs["record_tf"] is False + + +def test_openyam_teach_collection_is_a_minimal_native_stack() -> None: + modules = [atom.module for atom in learning_collect_teach_openyam.active_blueprints] + + assert modules == [ + NativeCollectionRecorder, + EpisodeMonitorModule, + ControlCoordinator, + CameraModule, + ] + + +def test_openyam_teach_collection_uses_gravity_compensation_and_zero_stiffness() -> None: + coordinator = next( + atom + for atom in learning_collect_teach_openyam.active_blueprints + if atom.module is ControlCoordinator + ) + hardware = coordinator.kwargs["hardware"][0] + assert hardware.joints == OPENYAM_JOINTS + assert hardware.wb_config is not None + assert hardware.wb_config.kp == (0.0,) * len(OPENYAM_JOINTS) + assert hardware.wb_config.kd == (2.0, 2.0, 2.0, 0.5, 0.5, 0.5, 0.0) + if hardware.adapter_type == "openyam_damiao": + assert hardware.adapter_kwargs["runtime_config"].gravity_comp is True + assert hardware.adapter_kwargs["runtime_config"].passive_grippers == ("gripper",) + + tasks = coordinator.kwargs["tasks"] + assert [ + (task.name, task.type, task.joint_names, task.priority, task.params) for task in tasks + ] == [ + ( + "teach_openyam", + "trajectory", + OPENYAM_JOINTS, + 10, + {"hold_position_when_idle": True}, + ), + ] + + +def test_native_openyam_teach_paths_are_configurable_from_cli() -> None: + parsed = BlueprintConfigParser(learning_collect_teach_openyam).parse( + [ + "--nativecollectionrecorder.store.path", + "/tmp/native-openyam-teach.mcap", + "--WristCamera.hardware.camera-index", + "/dev/v4l/by-id/usb-wrist-camera", + "--task", + "place the cup", + ], + environ={}, + ) + + assert parsed.module_kwargs("nativecollectionrecorder")["store"]["path"] == ( + "/tmp/native-openyam-teach.mcap" + ) + assert parsed.module_kwargs("WristCamera")["hardware"]["camera_index"] == ( + "/dev/v4l/by-id/usb-wrist-camera" + ) + assert parsed.module_kwargs("episodemonitormodule")["task"] == "place the cup" diff --git a/dimos/robot/manipulators/openyam/learning.py b/dimos/robot/manipulators/openyam/learning.py index 61ae240d4c..f70839689e 100644 --- a/dimos/robot/manipulators/openyam/learning.py +++ b/dimos/robot/manipulators/openyam/learning.py @@ -31,7 +31,7 @@ class OpenYamLearningProfile(BaseConfig): - """One fixed observation/action schema for OpenYAM learning.""" + """One fixed observation/action schema for an OpenYAM collection mode.""" robot_type: str = "openyam" joint_names: tuple[str, ...] = tuple(OPENYAM_JOINTS) @@ -43,6 +43,7 @@ class OpenYamLearningProfile(BaseConfig): camera_frame_prefix: str = "wrist" camera_frame_id: str = "wrist_camera_link" image_feature: str = "observation.images.wrist" + action_stream: str = "applied_joint_position_command" repo_id: str = "local/openyam-wrist" def dataprep_config(self) -> DataPrepConfig: @@ -68,7 +69,7 @@ def dataprep_config(self) -> DataPrepConfig: }, action={ "action": FeatureSpec( - stream="applied_joint_position_command", + stream=self.action_stream, field="position", dtype="float32", shape=(len(joint_names),), @@ -95,3 +96,4 @@ def dataprep_config(self) -> DataPrepConfig: OPENYAM_LEARNING_PROFILE = OpenYamLearningProfile() +OPENYAM_TEACH_LEARNING_PROFILE = OpenYamLearningProfile(action_stream="coordinator_joint_state") diff --git a/dimos/robot/manipulators/openyam/test_learning.py b/dimos/robot/manipulators/openyam/test_learning.py index 91d5eae19e..f219e2bef5 100644 --- a/dimos/robot/manipulators/openyam/test_learning.py +++ b/dimos/robot/manipulators/openyam/test_learning.py @@ -13,7 +13,10 @@ # limitations under the License. from dimos.robot.manipulators.openyam.config import OPENYAM_JOINTS -from dimos.robot.manipulators.openyam.learning import OPENYAM_LEARNING_PROFILE +from dimos.robot.manipulators.openyam.learning import ( + OPENYAM_LEARNING_PROFILE, + OPENYAM_TEACH_LEARNING_PROFILE, +) def test_openyam_profile_builds_matching_observation_and_action_schema() -> None: @@ -27,3 +30,13 @@ def test_openyam_profile_builds_matching_observation_and_action_schema() -> None assert config.action["action"].names == OPENYAM_JOINTS assert config.observation[profile.image_feature].shape == (480, 640, 3) assert config.output.metadata["robot_type"] == "openyam" + + +def test_openyam_teach_profile_uses_measured_joint_state_as_action() -> None: + profile = OPENYAM_TEACH_LEARNING_PROFILE + + config = profile.dataprep_config() + + assert config.observation["observation.state"].names == OPENYAM_JOINTS + assert config.action["action"].names == OPENYAM_JOINTS + assert config.action["action"].stream == "coordinator_joint_state" diff --git a/examples/native-modules/rust/Cargo.toml b/examples/native-modules/rust/Cargo.toml index d2c1d48d48..9ee0334293 100644 --- a/examples/native-modules/rust/Cargo.toml +++ b/examples/native-modules/rust/Cargo.toml @@ -21,7 +21,7 @@ path = "src/tf_broadcaster.rs" [dependencies] dimos-module = { path = "../../../native/rust/dimos-module" } -lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" } +lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", rev = "dd2159513ebfaa7ebc5fc32bf60209cd09aa1ca5" } tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } serde = { version = "1", features = ["derive"] } tracing = "0.1"