feat(drivers): agent-desktop on Windows, on a new WindowsDriver in the protocol

packages/agentenv-protocol/pyproject.toml

@@ -61,6 +61,10 @@ x11 = [
     "mss>=9,<11",
     "python-xlib>=0.33,<0.34",
 ]
+# A Windows driver on WindowsDriver: mss captures the desktop and SendInput, through ctypes, sends its input.
+windows = [
+    "mss>=9,<11",
+]
 dev = [
     "a2a-sdk[http-server]>=0.3.26,<0.4",
     "python-multipart>=0.0.13",

packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py

@@ -100,18 +100,20 @@
     )
     from .driver import DEFAULT_IDENTITY, ComputerDriverEnvironment, DriverDeclaration, computer_driver
     from ._macos import MacOSDriver
+    from ._windows import WindowsDriver
     from ._x11 import X11Driver
     from .recording import RecordedFrame, RecordedSegment, RecordHints, Recording, frames_in_thread
 
 # The names whose modules need the computer extra (mcp, av and Pillow), by module; MacOSDriver's, the macos extra too,
-# on first use of the Mac, and X11Driver's, the x11 extra.
+# on first use of the Mac, X11Driver's, the x11 extra, and WindowsDriver's, the windows extra.
 _ON_USE = {
     "driver": ("ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", "DriverDeclaration"),
     "card": ("COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER",
              "COMPUTER_KEYBOARD", "COMPUTER_PAGE", "COMPUTER_EXEC", "COMPUTER_RECORDING"),
     "recording": ("Recording", "RecordHints", "RecordedFrame", "RecordedSegment", "frames_in_thread"),
     "_macos": ("MacOSDriver",),
     "_x11": ("X11Driver",),
+    "_windows": ("WindowsDriver",),
 }
 
 
@@ -124,7 +126,7 @@ def __getattr__(name: str) -> Any:
 
 __all__ = [
     "ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", "DriverDeclaration", "MacOSDriver",
-    "X11Driver",
+    "X11Driver", "WindowsDriver",
     "ComputerAppEnvironment", "computer_app", "install", "linux", "windows", "install_step", "ready_check", "Linux",
     "Windows", "Service",
     "Launch", "Ready", "Packages", "AptSource", "Download", "RunScript",

packages/agentenv-protocol/src/agentenv_protocol/computer/_desktop.py

@@ -0,0 +1,35 @@
+"""What the drivers on an OS's own input share: the pacing of clicks and drags, how many presses and points they take,
+a drag's path, and a capture of every monitor through mss, whose class each driver passes from its own import."""
+from __future__ import annotations
+
+import math
+from typing import Any
+
+from .errors import DeviceUnavailable
+from .operations import Image
+
+MAX_CLICK_COUNT = 3
+MAX_DRAG_POINTS = 50
+#: The pause between the presses of a double or triple click, in seconds.
+CLICK_GAP_S = 0.04
+#: How far apart a drag's pointer moves are, in pixels.
+DRAG_STEP_PX = 20
+
+
+def drag_steps(path: list) -> list[tuple[float, float]]:
+    """The points a drag moves through after its start, about every DRAG_STEP_PX pixels."""
+    steps = []
+    for (x0, y0), (x1, y1) in zip(path, path[1:]):
+        n = max(2, int(math.hypot(x1 - x0, y1 - y0) / DRAG_STEP_PX))
+        steps += [(x0 + (x1 - x0) * k / n, y0 + (y1 - y0) * k / n) for k in range(1, n + 1)]
+    return steps
+
+
+def mss_capture(mss_class: Any) -> Image:
+    """Every monitor, as mss's first monitor spans them, in BGRA."""
+    try:
+        with mss_class() as m:
+            shot = m.grab(m.monitors[0])
+            return Image(shot.width, shot.height, bytes(shot.bgra), "BGRA")
+    except Exception as e:
+        raise DeviceUnavailable(f"capture failed: {type(e).__name__}: {e}") from None

packages/agentenv-protocol/src/agentenv_protocol/computer/_macos.py

@@ -14,6 +14,7 @@
 from contextlib import contextmanager
 from typing import Any, Callable, Iterator
 
+from ._desktop import CLICK_GAP_S, DRAG_STEP_PX, MAX_CLICK_COUNT, MAX_DRAG_POINTS
 from .driver import ComputerDriverEnvironment
 from .errors import DeliveryFailed, DeviceUnavailable, Unsupported
 from .keys import PRINTABLE_KEYS
@@ -59,10 +60,6 @@
 SETTLE_S = 0.2
 #: How long a click holds its button down, as a hand does: a press released at once is lost now and then.
 CLICK_HOLD_S = 0.03
-#: The pause between the presses of a double or triple click.
-CLICK_GAP_S = 0.04
-#: How far apart a drag's pointer moves are, in pixels.
-DRAG_STEP_PX = 20
 #: The pause between a scroll's wheel notches, as a wheel turned by hand has: notches posted back to back reach an app
 #: as fewer.
 SCROLL_GAP_S = 0.05
@@ -81,8 +78,8 @@ class MacOSDriver(ComputerDriverEnvironment):
     released."""
 
     buttons = ("left", "right", "middle")
-    max_click_count = 3
-    max_drag_points = 50
+    max_click_count = MAX_CLICK_COUNT
+    max_drag_points = MAX_DRAG_POINTS
     keys = (*MODIFIERS, *NAMED, *PRINTABLE_KEYS)
 
     def __init__(self, settings: ComputerSettings | None = None, platform: Platform | None = None) -> None: