From f7ddcd46b0198ef38de2f5151da13f20bfe9048d Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Mon, 16 Feb 2026 23:23:20 +0000 Subject: [PATCH 1/5] initial setup to allow interactions between Murfey and smartem adds registration of acquisition and atlases with smartem --- src/murfey/client/analyser.py | 6 ++- src/murfey/client/contexts/atlas.py | 33 +++++++++++++++ src/murfey/client/instance_environment.py | 1 + src/murfey/client/multigrid_control.py | 4 ++ src/murfey/instrument_server/api.py | 2 + src/murfey/server/api/instrument.py | 19 +++++++++ src/murfey/server/api/session_control.py | 49 +++++++++++++++++++++++ src/murfey/util/config.py | 1 + src/murfey/util/instrument_models.py | 2 + src/murfey/util/models.py | 1 + src/murfey/util/route_manifest.yaml | 7 ++++ 11 files changed, 124 insertions(+), 1 deletion(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index ba4925cf8..2d0fe7fd7 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -39,6 +39,7 @@ def __init__( environment: MurfeyInstanceEnvironment | None = None, force_mdoc_metadata: bool = False, limited: bool = False, + serialem: bool = False, ): super().__init__() self._basepath = basepath_local.absolute() @@ -52,6 +53,7 @@ def __init__( self._environment = environment self._force_mdoc_metadata = force_mdoc_metadata self._token = token + self._serialem = serialem self.parameters_model: ( Type[ProcessingParametersSPA] | Type[ProcessingParametersTomo] | None ) = None @@ -138,7 +140,9 @@ def _find_context(self, file_path: Path) -> bool: # Tomography and SPA workflow checks if "atlas" in file_path.parts: - self._context = AtlasContext("epu", self._basepath, self._token) + self._context = AtlasContext( + "serialem" if self._serialem else "epu", self._basepath, self._token + ) return True if "Metadata" in file_path.parts or file_path.name == "EpuSession.dm": diff --git a/src/murfey/client/contexts/atlas.py b/src/murfey/client/contexts/atlas.py index d44d3c580..b67dcac99 100644 --- a/src/murfey/client/contexts/atlas.py +++ b/src/murfey/client/contexts/atlas.py @@ -28,7 +28,40 @@ def post_transfer( environment=environment, **kwargs, ) + if self._acquisition_software == "serialem": + self.post_transfer_serialem( + transferred_file, environment=environment, **kwargs + ) + else: + self.post_transfer_epu(transferred_file, environment=environment, **kwargs) + def post_transfer_serialem( + self, + transferred_file: Path, + environment: Optional[MurfeyInstanceEnvironment] = None, + **kwargs, + ): + if environment and transferred_file.suffix == ".mrc": + source = _get_source(transferred_file, environment) + if source: + capture_post( + base_url=str(environment.url.geturl()), + router_name="session_control.spa_router", + function_name="register_atlas", + token=self._token, + session_id=environment.murfey_session, + data={ + "name": transferred_file.stem, + "acquisition_uuid": environment.acquisition_uuid, + }, + ) + + def post_transfer_epu( + self, + transferred_file: Path, + environment: Optional[MurfeyInstanceEnvironment] = None, + **kwargs, + ): if ( environment and "Atlas_" in transferred_file.stem diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index ecd33c49d..64bf40c82 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -53,6 +53,7 @@ class MurfeyInstanceEnvironment(BaseModel): murfey_session: Optional[int] = None samples: Dict[Path, SampleInfo] = {} rsync_url: str = "" + acquisition_uuid: Optional[str] = None model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index 173e49273..6bebbdfac 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -48,6 +48,8 @@ class MultigridController: analysers: Dict[Path, Analyser] = field(default_factory=lambda: {}) data_collection_parameters: dict = field(default_factory=lambda: {}) token: str = "" + serialem: bool = False + acquisition_uuid: Optional[str] = None _machine_config: dict = field(default_factory=lambda: {}) visit_end_time: Optional[datetime] = None @@ -72,6 +74,7 @@ def __post_init__(self): symmetry=self.data_collection_parameters.get("symmetry"), eer_fractionation=self.data_collection_parameters.get("eer_fractionation"), instrument_name=self.instrument_name, + acquisition_uuid=self.acquisition_uuid, ) self._machine_config = get_machine_config_client( str(self._environment.url.geturl()), @@ -449,6 +452,7 @@ def rsync_result(update: RSyncerUpdate): environment=self._environment if not self.dummy_dc else None, force_mdoc_metadata=self.force_mdoc_metadata, limited=limited, + serialem=self.serialem, ) self.analysers[source].subscribe(self._start_dc) self.analysers[source].start() diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index cd968584b..552a1c9a1 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -179,6 +179,8 @@ def setup_multigrid_watcher( data_collection_parameters=data_collection_parameters.get(label, {}), rsync_restarts=watcher_spec.rsync_restarts, visit_end_time=watcher_spec.visit_end_time, + acquisition_uuid=watcher_spec.acquisition_uuid, + serialem=watcher_spec.serialem, ) # Make child directories, if specified watcher_spec.source.mkdir(exist_ok=True) diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 35f4ccebb..8044bd10c 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -13,6 +13,13 @@ from sqlmodel import select from werkzeug.utils import secure_filename +try: + from smartem_common.schemas import AcquisitionData + + SMARTEM_ACTIVE = True +except ImportError: + SMARTEM_ACTIVE = False + import murfey.server.prometheus as prom from murfey.server.api.auth import ( MurfeyInstrumentNameFrontend as MurfeyInstrumentName, @@ -75,6 +82,7 @@ async def activate_instrument_server_for_session( success = response.status == 200 instrument_server_token = await response.json() instrument_server_tokens[session_id] = instrument_server_token + if success: log.info("Handshake successful") else: @@ -147,6 +155,15 @@ async def setup_multigrid_watcher( session = db.exec(select(Session).where(Session.id == session_id)).one() visit = session.visit async with aiohttp.ClientSession() as clientsession: + acquisition_uuid = None + if SMARTEM_ACTIVE and machine_config.smartem_api_url: + acquisition_data = AcquisitionData(name=visit) + async with clientsession.post( + f"{machine_config.smartem_api_url}/acquisitions", + AcquisitionData.model_json_schema(), + ) as response: + acquisition_data = await response.json() + acquisition_uuid = acquisition_data.uuid async with clientsession.post( f"{machine_config.instrument_server_url}{url_path_for('api.router', 'setup_multigrid_watcher', session_id=session_id)}", json={ @@ -161,6 +178,8 @@ async def setup_multigrid_watcher( "visit_end_time": ( str(session.visit_end_time) if session.visit_end_time else None ), + "acquisition_uuid": acquisition_uuid, + "serialem": watcher_spec.serialem, }, headers={ "Authorization": f"Bearer {instrument_server_tokens[session_id]['access_token']}" diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index c384fb62d..67d4ded42 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -10,6 +10,14 @@ from sqlalchemy import func from sqlmodel import select +try: + from smartem_backend.api_client import SmartEMAPIClient + from smartem_common.schemas import AtlasData + + SMARTEM_ACTIVE = True +except ImportError: + SMARTEM_ACTIVE = False + import murfey.server.prometheus as prom from murfey.server import _transport_object from murfey.server.api.auth import ( @@ -349,6 +357,47 @@ def get_foil_hole( return _get_foil_hole(session_id, fh_name, db) +class AtlasRegistration(BaseModel): + name: str + acqusition_uuid: str + + +@spa_router.post("/sessions/{session_id}/register_atlas") +def register_atlas( + session_id: MurfeySessionID, + atlas_registration_data: AtlasRegistration, + db=murfey_db, +): + if SMARTEM_ACTIVE: + session = db.exec(select(Session).where(Session.id == session_id)).one() + machine_config = get_machine_config(session.instrument_name)[ + session.instrument_name + ] + if machine_config.smartem_api_url: + smartem_client = SmartEMAPIClient( + base_url=machine_config.smartem_api_url, logger=logger + ) + possible_grids = smartem_client.get_acquisition_grids( + atlas_registration_data.acqusition_uuid + ) + grid_uuid = None + for grid in possible_grids: + if grid.name == atlas_registration_data.name.replace("_atlas", ""): + grid_uuid = grid.uuid + break + if grid_uuid is not None: + atlas_data = AtlasData( + id=atlas_registration_data.name, + acquisition_data=datetime.now(), + storage_folder="", + name=atlas_registration_data.name, + tiles=[], + gridsquare_positions=None, + grid_uuid=grid_uuid, + ) + smartem_client.create_grid_atlas(atlas_data) + + @spa_router.post("/sessions/{session_id}/make_atlas_jpg") def make_atlas_jpg( session_id: MurfeySessionID, atlas_mrc: StringOfPathModel, db=murfey_db diff --git a/src/murfey/util/config.py b/src/murfey/util/config.py index c0d62beb7..5e50d7c22 100644 --- a/src/murfey/util/config.py +++ b/src/murfey/util/config.py @@ -106,6 +106,7 @@ class MachineConfig(BaseModel): # type: ignore murfey_url: str = "http://localhost:8000" frontend_url: str = "http://localhost:3000" instrument_server_url: str = "http://localhost:8001" + smartem_api_url: str = "" # Messaging queues failure_queue: str = "" diff --git a/src/murfey/util/instrument_models.py b/src/murfey/util/instrument_models.py index ff1c16e11..2804ed69a 100644 --- a/src/murfey/util/instrument_models.py +++ b/src/murfey/util/instrument_models.py @@ -13,3 +13,5 @@ class MultigridWatcherSpec(BaseModel): destination_overrides: Dict[Path, str] = {} rsync_restarts: List[str] = [] visit_end_time: Optional[datetime] = None + acquisition_uuid: Optional[str] = None + serialem: bool = False diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 1d59a6b25..c5c348ee6 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -208,6 +208,7 @@ class MultigridWatcherSetup(BaseModel): source: Path destination_overrides: Dict[Path, str] = {} rsync_restarts: List[str] = [] + serialem: bool = False class Token(BaseModel): diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index 6fdeaf47f..091260b33 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -975,6 +975,13 @@ murfey.server.api.session_control.spa_router: type: int methods: - GET + - path: /session_control/spa/sessions/{session_id}/register_atlas + function: register_atlas + path_params: + - name: session_id + type: int + methods: + - POST - path: /session_control/spa/sessions/{session_id}/make_atlas_jpg function: make_atlas_jpg path_params: From b99a9ac73763a6414650e4208b5ee40973d364a3 Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Mon, 16 Feb 2026 23:27:13 +0000 Subject: [PATCH 2/5] add endpoint to check if smartem has been configured --- src/murfey/server/api/session_info.py | 6 ++++++ src/murfey/util/route_manifest.yaml | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/murfey/server/api/session_info.py b/src/murfey/server/api/session_info.py index d43dd1d8d..64bde37fa 100644 --- a/src/murfey/server/api/session_info.py +++ b/src/murfey/server/api/session_info.py @@ -81,6 +81,12 @@ def machine_info_by_instrument( return get_machine_config(instrument_name)[instrument_name] +@router.get("/instruments/{instrument_name}/smartem") +def check_smartem_availability(instrument_name: str): + machine_config = get_machine_config(instrument_name)[instrument_name] + return {"available": bool(machine_config.smartem_api_url)} + + @router.get("/instruments/{instrument_name}/visits_raw", response_model=List[Visit]) def get_current_visits(instrument_name: MurfeyInstrumentName, db=ispyb_db): logger.debug( diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index 091260b33..9858cb800 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -1092,6 +1092,13 @@ murfey.server.api.session_info.router: type: str methods: - GET + - path: /session_info/instruments/{instrument_name}/smartem + function: check_smartem_availability + path_params: + - name: instrument_name + type: str + methods: + - GET - path: /session_info/instruments/{instrument_name}/visits_raw function: get_current_visits path_params: From d0dbb4882ea0102b052942db50a455ba87e75e85 Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Tue, 17 Feb 2026 09:46:52 +0000 Subject: [PATCH 3/5] add optional smartem-decisions dependency --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index de25caddd..3bf91616b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ server = [ "stomp-py>8.1.1", # 8.1.1 (released 2024-04-06) doesn't work with our project "zocalo>=1", ] +smartem = [ + "smartem-decisions[backend]", +] [project.urls] Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues" Documentation = "https://github.com/DiamondLightSource/python-murfey" From 38becc08723409bccdd4b3d17cf3d640312abf60 Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Tue, 17 Feb 2026 12:32:10 +0000 Subject: [PATCH 4/5] variable naming confusion --- src/murfey/server/api/instrument.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 8044bd10c..27c6b2b0b 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -160,10 +160,10 @@ async def setup_multigrid_watcher( acquisition_data = AcquisitionData(name=visit) async with clientsession.post( f"{machine_config.smartem_api_url}/acquisitions", - AcquisitionData.model_json_schema(), + acquisition_data.model_json_schema(), ) as response: - acquisition_data = await response.json() - acquisition_uuid = acquisition_data.uuid + acquisition_response_data = await response.json() + acquisition_uuid = acquisition_response_data.uuid async with clientsession.post( f"{machine_config.instrument_server_url}{url_path_for('api.router', 'setup_multigrid_watcher', session_id=session_id)}", json={ From 86bee44d4e3a33db93e4b2edb3cb4714bb3c7b17 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Wed, 18 Feb 2026 09:58:14 +0000 Subject: [PATCH 5/5] add logs and minor fix --- src/murfey/server/api/instrument.py | 25 +++++++++++++++++------- src/murfey/server/api/session_control.py | 2 ++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 27c6b2b0b..a7752f824 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -14,6 +14,7 @@ from werkzeug.utils import secure_filename try: + from smartem_backend.api_client import EntityConverter from smartem_common.schemas import AcquisitionData SMARTEM_ACTIVE = True @@ -157,13 +158,23 @@ async def setup_multigrid_watcher( async with aiohttp.ClientSession() as clientsession: acquisition_uuid = None if SMARTEM_ACTIVE and machine_config.smartem_api_url: - acquisition_data = AcquisitionData(name=visit) - async with clientsession.post( - f"{machine_config.smartem_api_url}/acquisitions", - acquisition_data.model_json_schema(), - ) as response: - acquisition_response_data = await response.json() - acquisition_uuid = acquisition_response_data.uuid + log.info("registering an acquisition with smartem") + try: + acquisition_data = EntityConverter.acquisition_to_request( + AcquisitionData(name=visit) + ) + async with clientsession.post( + f"{machine_config.smartem_api_url}/acquisitions", + json=acquisition_data.model_dump(), + ) as response: + acquisition_response_data = await response.json() + acquisition_uuid = acquisition_response_data["uuid"] + except Exception: + log.warning( + "failed to register acquisition with smartem", exc_info=True + ) + else: + log.info("smartem not configured") async with clientsession.post( f"{machine_config.instrument_server_url}{url_path_for('api.router', 'setup_multigrid_watcher', session_id=session_id)}", json={ diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index 67d4ded42..45f9cdeae 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -396,6 +396,8 @@ def register_atlas( grid_uuid=grid_uuid, ) smartem_client.create_grid_atlas(atlas_data) + else: + logger.info("smartem deactivated so did not register atlas") @spa_router.post("/sessions/{session_id}/make_atlas_jpg")