From 9cb3227420c93144d3cfcf33bc936dc62439b6f1 Mon Sep 17 00:00:00 2001 From: P S Kesavan Date: Mon, 29 Jun 2026 13:22:31 +0530 Subject: [PATCH] Make the Python client work on Linux The external Python API previously assumed Windows transports (named pipe + named file mapping). This makes both the client and the server side of the local API cross-platform, so a running ScopeOne can be driven from Python on Linux. Windows behaviour is unchanged (guarded by _WIN32 / os.name). Server (src/ScopeOneLocalApiServer.cpp/.h): - Platform-aware control endpoint: keep the Windows named pipe, use a plain QLocalServer name on Unix (resolves to /ScopeOne.Api.local). - Frame export on Linux via a POSIX shared-memory object (shm_open + mmap), visible to external clients at /dev/shm/ScopeOne.Api.frame; cleaned up with munmap/close/shm_unlink on shutdown. - Frame-export readiness check is platform-specific: the Windows handle+view check is left unchanged; Linux checks the mapped view (there is no Windows handle there). Client (ScopeOneCore/python/scopeone): - Split the transport: Windows keeps the pywin32 pipe + tagname mmap; Unix uses an AF_UNIX socket and reads frames by mmapping /dev/shm. Protocol and frame layout are identical, so the public API and shm.py are unchanged. - pyproject: make pywin32 a Windows-only dependency and add the Linux classifier. Verified on Linux against the demo camera: connect -> load_config -> preview -> record -> session.frame() returns a (512,512) uint16 numpy array. --- ScopeOneCore/python/scopeone/pyproject.toml | 3 +- .../python/scopeone/src/scopeone/client.py | 232 +++++++++++++----- src/ScopeOneLocalApiServer.cpp | 56 +++++ src/ScopeOneLocalApiServer.h | 1 + 4 files changed, 234 insertions(+), 58 deletions(-) diff --git a/ScopeOneCore/python/scopeone/pyproject.toml b/ScopeOneCore/python/scopeone/pyproject.toml index 49821ee..7d39b76 100644 --- a/ScopeOneCore/python/scopeone/pyproject.toml +++ b/ScopeOneCore/python/scopeone/pyproject.toml @@ -18,6 +18,7 @@ classifiers = [ "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -30,7 +31,7 @@ classifiers = [ dependencies = [ "numpy>=1.20.0", - "pywin32>=300", + "pywin32>=300; platform_system == 'Windows'", ] [tool.setuptools] diff --git a/ScopeOneCore/python/scopeone/src/scopeone/client.py b/ScopeOneCore/python/scopeone/src/scopeone/client.py index b305285..56871c2 100644 --- a/ScopeOneCore/python/scopeone/src/scopeone/client.py +++ b/ScopeOneCore/python/scopeone/src/scopeone/client.py @@ -1,27 +1,181 @@ -"""Backend clients for the public ScopeOne facade.""" +"""Backend clients for the public ScopeOne facade. + +The control channel and frame transport are platform-specific: on Windows the +server exposes a named pipe plus a named file mapping, and on Linux/Unix a +QLocalServer unix socket plus a POSIX shared-memory object under /dev/shm. The +JSON request protocol and the shared-frame layout are identical on both. +""" from __future__ import annotations import json import mmap +import os import struct import time -try: - import pywintypes - import win32file -except ImportError: - pywintypes = None - win32file = None +_IS_WINDOWS = os.name == "nt" + +if _IS_WINDOWS: + try: + import pywintypes + import win32file + except ImportError: + pywintypes = None + win32file = None +else: + import socket + +# Windows: named pipe. Unix: QLocalServer turns this name into a socket file +# under the temp directory (e.g. /tmp/ScopeOne.Api.local). +WIN_LOCAL_SERVER_NAME = r"\\.\pipe\ScopeOne.Api.local" +UNIX_LOCAL_SERVER_NAME = "ScopeOne.Api.local" +LOCAL_SERVER_NAME = WIN_LOCAL_SERVER_NAME if _IS_WINDOWS else UNIX_LOCAL_SERVER_NAME -LOCAL_SERVER_NAME = r"\\.\pipe\ScopeOne.Api.local" MAX_MESSAGE_BYTES = 256 * 1024 +_CONNECT_TIMEOUT_S = 5.0 -class ExternalClient: - def __init__(self, server_name: str = LOCAL_SERVER_NAME) -> None: + +class _WindowsPipeTransport: + """Control channel + frame mapping over Windows named pipe / file mapping.""" + + def __init__(self, server_name: str) -> None: if pywintypes is None or win32file is None: raise RuntimeError("pywin32 is required for ScopeOne external client.") - self._handle = self._connect_pipe(server_name) + self._handle = self._connect(server_name) + + @staticmethod + def _pipe_path(server_name: str) -> str: + if server_name.startswith("\\\\.\\pipe\\"): + return server_name + return rf"\\.\pipe\{server_name}" + + def _connect(self, server_name: str): + pipe_path = self._pipe_path(server_name) + deadline = time.monotonic() + _CONNECT_TIMEOUT_S + while True: + try: + return win32file.CreateFile( + pipe_path, + win32file.GENERIC_READ | win32file.GENERIC_WRITE, + 0, + None, + win32file.OPEN_EXISTING, + 0, + None, + ) + except pywintypes.error as exc: + if time.monotonic() >= deadline: + raise RuntimeError( + f"Failed to connect to ScopeOne server '{server_name}': {exc}" + ) from exc + time.sleep(0.05) + + def send(self, data: bytes) -> None: + try: + win32file.WriteFile(self._handle, data) + except pywintypes.error as exc: + raise RuntimeError(f"ScopeOne control request failed: {exc}") from exc + + def recv_exact(self, size: int) -> bytes: + chunks = bytearray() + while len(chunks) < size: + try: + _, data = win32file.ReadFile(self._handle, size - len(chunks)) + except pywintypes.error as exc: + raise RuntimeError(f"ScopeOne control request failed: {exc}") from exc + if not data: + raise RuntimeError("ScopeOne control connection closed") + chunks.extend(data) + return bytes(chunks) + + def open_frame(self, mapping_name: str, mapping_size: int): + return mmap.mmap(-1, mapping_size, tagname=mapping_name, access=mmap.ACCESS_READ) + + def close(self) -> None: + if self._handle is not None: + try: + win32file.CloseHandle(self._handle) + except Exception: + pass + self._handle = None + + +class _UnixSocketTransport: + """Control channel over unix socket + frame read from /dev/shm.""" + + def __init__(self, server_name: str) -> None: + self._sock = self._connect(server_name) + + @staticmethod + def _server_path(server_name: str) -> str: + if os.path.isabs(server_name): + return server_name + # Matches Qt's QDir::tempPath() (honors $TMPDIR, else /tmp). + return os.path.join(os.environ.get("TMPDIR", "/tmp"), server_name) + + def _connect(self, server_name: str): + path = self._server_path(server_name) + deadline = time.monotonic() + _CONNECT_TIMEOUT_S + while True: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + sock.connect(path) + return sock + except OSError as exc: + sock.close() + if time.monotonic() >= deadline: + raise RuntimeError( + f"Failed to connect to ScopeOne server '{path}': {exc}" + ) from exc + time.sleep(0.05) + + def send(self, data: bytes) -> None: + try: + self._sock.sendall(data) + except OSError as exc: + raise RuntimeError(f"ScopeOne control request failed: {exc}") from exc + + def recv_exact(self, size: int) -> bytes: + chunks = bytearray() + while len(chunks) < size: + try: + data = self._sock.recv(size - len(chunks)) + except OSError as exc: + raise RuntimeError(f"ScopeOne control request failed: {exc}") from exc + if not data: + raise RuntimeError("ScopeOne control connection closed") + chunks.extend(data) + return bytes(chunks) + + def open_frame(self, mapping_name: str, mapping_size: int): + # The server publishes the frame as a POSIX shm object; it appears at + # /dev/shm/. mapping_name is the bare object name. + path = mapping_name if os.path.isabs(mapping_name) else f"/dev/shm/{mapping_name}" + fd = os.open(path, os.O_RDONLY) + try: + return mmap.mmap(fd, mapping_size, prot=mmap.PROT_READ) + finally: + os.close(fd) + + def close(self) -> None: + if self._sock is not None: + try: + self._sock.close() + except Exception: + pass + self._sock = None + + +def _make_transport(server_name: str): + if _IS_WINDOWS: + return _WindowsPipeTransport(server_name) + return _UnixSocketTransport(server_name) + + +class ExternalClient: + def __init__(self, server_name: str = LOCAL_SERVER_NAME) -> None: + self._transport = _make_transport(server_name) self._request({"type": "ping"}) def load_config(self, config_path: str) -> bool: @@ -174,33 +328,8 @@ def record( response = self._request(request) return ExternalRecordingSession(self, str(response["sessionId"])) - @staticmethod - def _pipe_path(server_name: str) -> str: - if server_name.startswith("\\\\.\\pipe\\"): - return server_name - return rf"\\.\pipe\{server_name}" - - @classmethod - def _connect_pipe(cls, server_name: str): - pipe_path = cls._pipe_path(server_name) - deadline = time.monotonic() + 5.0 - while True: - try: - return win32file.CreateFile( - pipe_path, - win32file.GENERIC_READ | win32file.GENERIC_WRITE, - 0, - None, - win32file.OPEN_EXISTING, - 0, - None, - ) - except pywintypes.error as exc: - if time.monotonic() >= deadline: - raise RuntimeError( - f"Failed to connect to ScopeOne server '{server_name}': {exc}" - ) from exc - time.sleep(0.05) + def close(self) -> None: + self._transport.close() def _request(self, message: dict): payload = json.dumps(message, separators=(",", ":")).encode("utf-8") @@ -208,30 +337,19 @@ def _request(self, message: dict): raise RuntimeError("ScopeOne control message is invalid or too large") framed = struct.pack(" MAX_MESSAGE_BYTES: - raise RuntimeError("ScopeOne control response has invalid size") - response = json.loads(self._read_exact(payload_size).decode("utf-8")) - except pywintypes.error as exc: - raise RuntimeError(f"ScopeOne control request failed: {exc}") from exc + self._transport.send(framed) + + response_size = self._transport.recv_exact(4) + payload_size = struct.unpack(" MAX_MESSAGE_BYTES: + raise RuntimeError("ScopeOne control response has invalid size") + response = json.loads(self._transport.recv_exact(payload_size).decode("utf-8")) if not response.get("ok", False): error = response.get("error", "ScopeOne request failed") raise RuntimeError(error) return response - def _read_exact(self, size: int) -> bytes: - chunks = bytearray() - while len(chunks) < size: - _, data = win32file.ReadFile(self._handle, size - len(chunks)) - if not data: - raise RuntimeError("ScopeOne control connection closed") - chunks.extend(data) - return bytes(chunks) - class ExternalRecordingSession: def __init__(self, client: ExternalClient, session_id: str) -> None: @@ -268,7 +386,7 @@ def frame(self, camera: str, index: int): ) mapping_name = str(response["mappingName"]) mapping_size = int(response["mappingSize"]) - with mmap.mmap(-1, mapping_size, tagname=mapping_name, access=mmap.ACCESS_READ) as view: + with self._client._transport.open_frame(mapping_name, mapping_size) as view: header = parse_frame_header(view[:SHARED_FRAME_HEADER_SIZE]) return frame_to_ndarray(header, view[SHARED_FRAME_HEADER_SIZE:]) diff --git a/src/ScopeOneLocalApiServer.cpp b/src/ScopeOneLocalApiServer.cpp index 9fdc228..c1cb777 100644 --- a/src/ScopeOneLocalApiServer.cpp +++ b/src/ScopeOneLocalApiServer.cpp @@ -18,6 +18,10 @@ #if defined(_WIN32) #include +#else +#include +#include +#include #endif namespace scopeone::ui @@ -25,7 +29,14 @@ namespace scopeone::ui namespace { constexpr quint32 kMaxMessageBytes = 256 * 1024; +#if defined(_WIN32) const QString kServerName = QStringLiteral(R"(\\.\pipe\ScopeOne.Api.local)"); +#else + // QLocalServer turns this into a unix socket at /ScopeOne.Api.local + const QString kServerName = QStringLiteral("ScopeOne.Api.local"); + // POSIX shared memory object, visible to external clients at /dev/shm/ScopeOne.Api.frame + const char* const kPosixFrameShmName = "/ScopeOne.Api.frame"; +#endif const QString kFrameMappingName = QStringLiteral("ScopeOne.Api.frame"); // Encodes one JSON object with a little endian size prefix @@ -247,6 +258,33 @@ namespace scopeone::ui CloseHandle(m_frameMappingHandle); m_frameMappingHandle = nullptr; } +#else + const size_t mappingSize = static_cast( + scopeone::core::kSharedFrameHeaderSize + scopeone::core::kSharedFrameMaxBytes); + m_frameShmFd = ::shm_open(kPosixFrameShmName, O_CREAT | O_RDWR, 0600); + if (m_frameShmFd < 0) + { + qWarning().noquote() << QStringLiteral("ScopeOne API frame shm create failed"); + return; + } + if (::ftruncate(m_frameShmFd, static_cast(mappingSize)) != 0) + { + qWarning().noquote() << QStringLiteral("ScopeOne API frame shm resize failed"); + ::close(m_frameShmFd); + m_frameShmFd = -1; + ::shm_unlink(kPosixFrameShmName); + return; + } + void* view = ::mmap(nullptr, mappingSize, PROT_READ | PROT_WRITE, MAP_SHARED, m_frameShmFd, 0); + if (view == MAP_FAILED) + { + qWarning().noquote() << QStringLiteral("ScopeOne API frame shm map failed"); + ::close(m_frameShmFd); + m_frameShmFd = -1; + ::shm_unlink(kPosixFrameShmName); + return; + } + m_frameMappingView = static_cast(view); #endif } @@ -264,6 +302,20 @@ namespace scopeone::ui CloseHandle(m_frameMappingHandle); m_frameMappingHandle = nullptr; } +#else + if (m_frameMappingView) + { + const size_t mappingSize = static_cast( + scopeone::core::kSharedFrameHeaderSize + scopeone::core::kSharedFrameMaxBytes); + ::munmap(m_frameMappingView, mappingSize); + m_frameMappingView = nullptr; + } + if (m_frameShmFd >= 0) + { + ::close(m_frameShmFd); + m_frameShmFd = -1; + ::shm_unlink(kPosixFrameShmName); + } #endif } @@ -858,7 +910,11 @@ namespace scopeone::ui const scopeone::core::ScopeOneCore::RecordingFrame& frame, QString& errorMessage) { +#if defined(_WIN32) if (!m_frameMappingHandle || !m_frameMappingView) +#else + if (!m_frameMappingView) +#endif { errorMessage = QStringLiteral("Frame mapping is not available"); return false; diff --git a/src/ScopeOneLocalApiServer.h b/src/ScopeOneLocalApiServer.h index c9c81ce..9fcad85 100644 --- a/src/ScopeOneLocalApiServer.h +++ b/src/ScopeOneLocalApiServer.h @@ -42,5 +42,6 @@ namespace scopeone::ui void* m_frameMappingHandle{nullptr}; uchar* m_frameMappingView{nullptr}; + int m_frameShmFd{-1}; }; } // namespace scopeone::ui