From 541534f06ab69c9a6bf275fa3a663baec26a6d64 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 6 Feb 2025 13:47:13 +0000 Subject: [PATCH 1/5] Get the longest source, and log failed Images-Disc searches --- src/murfey/client/contexts/spa.py | 13 ++++++++-- src/murfey/client/contexts/spa_metadata.py | 30 ++++++++++++---------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 9508108c6..1bfa7a5d7 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -79,10 +79,19 @@ def _grid_square_metadata_file( def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None: + possible_sources = [] for s in environment.sources: if file_path.is_relative_to(s): - return s - return None + possible_sources.append(s) + if not possible_sources: + return None + elif len(possible_sources) == 1: + return possible_sources[0] + source = possible_sources[0] + for extra_source in possible_sources[1:]: + if extra_source.is_relative_to(source): + source = extra_source + return source def _get_xml_list_index(key: str, xml_list: list) -> int: diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 993530786..120a5ca87 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -151,16 +151,16 @@ def post_transfer( ) url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group" dcg_search_dir = "/".join( - p - for p in transferred_file.parent.parts[1:] - if p != environment.visit + p for p in transferred_file.parent.parts if p != environment.visit ) - dcg_tag = str( - sorted( - Path(dcg_search_dir).glob("Images-Disc*"), - key=lambda x: x.stat().st_ctime, - )[-1] + dcg_images_dirs = sorted( + Path(dcg_search_dir).glob("Images-Disc*"), + key=lambda x: x.stat().st_ctime, ) + if not dcg_images_dirs: + logger.warning(f"Cannot find Images-Disc* in {dcg_search_dir}") + return + dcg_tag = str(dcg_images_dirs[-1]) dcg_data = { "experiment_type": "single particle", "experiment_type_id": 37, @@ -206,12 +206,16 @@ def post_transfer( visitless_source_search_dir = str(source).replace( f"/{environment.visit}", "" ) - visitless_source = str( - sorted( - Path(visitless_source_search_dir).glob("Images-Disc*"), - key=lambda x: x.stat().st_ctime, - )[-1] + visitless_source_images_dirs = sorted( + Path(visitless_source_search_dir).glob("Images-Disc*"), + key=lambda x: x.stat().st_ctime, ) + if not visitless_source_images_dirs: + logger.warning( + f"Cannot find Images-Disc* in {visitless_source_search_dir}" + ) + return + visitless_source = str(visitless_source_images_dirs[-1]) for fh, fh_data in fh_positions.items(): capture_post( f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/grid_square/{gs_name}/foil_hole", From f99b4928d11935d4765b028d3817c0d142f8da6b Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 6 Feb 2025 17:25:46 +0000 Subject: [PATCH 2/5] Various atlas fixes --- src/murfey/client/contexts/spa_metadata.py | 14 +++++++------- src/murfey/server/__init__.py | 1 + src/murfey/server/ispyb.py | 5 +++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 120a5ca87..db4b01547 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -117,12 +117,10 @@ def post_transfer( source_visit_dir = source.parent logger.info( - f"Looking for atlas XML file in metadata directory {str((source_visit_dir / environment.visit / partial_path).parent)}" + f"Looking for atlas XML file in metadata directory {str((source_visit_dir / partial_path).parent)}" ) atlas_xml_path = list( - (source_visit_dir / environment.visit / partial_path).parent.glob( - "Atlas_*.xml" - ) + (source_visit_dir / partial_path).parent.glob("Atlas_*.xml") )[0] logger.info(f"Atlas XML path {str(atlas_xml_path)} found") with open(atlas_xml_path, "rb") as atlas_xml: @@ -150,8 +148,10 @@ def post_transfer( atlas=Path(partial_path), sample=sample ) url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group" - dcg_search_dir = "/".join( - p for p in transferred_file.parent.parts if p != environment.visit + dcg_search_dir = "/" + "/".join( + p + for p in transferred_file.parent.parts[1:] + if p != environment.visit ) dcg_images_dirs = sorted( Path(dcg_search_dir).glob("Images-Disc*"), @@ -174,7 +174,7 @@ def post_transfer( } capture_post(url, json=dcg_data) gs_pix_positions = get_grid_square_atlas_positions( - source_visit_dir / environment.visit / partial_path + source_visit_dir / partial_path ) for gs, pos_data in gs_pix_positions.items(): if pos_data: diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index b22c80acb..fa5745eb7 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -2569,6 +2569,7 @@ def feedback_callback(header: dict, message: dict) -> None: message["atlas_pixel_size"], message["sample"], ) + _transport_object.transport.ack(header) return None elif message["register"] == "data_collection": murfey_session_id = message["session_id"] diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index d07342ce9..6b6aa8ed2 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -213,7 +213,8 @@ def do_update_grid_square( grid_square_parameters.readout_area_x / grid_square_parameters.thumbnail_size_x ) - grid_square.gridSquareImage = grid_square_parameters.image + if grid_square_parameters.image: + grid_square.gridSquareImage = grid_square_parameters.image grid_square.pixelLocationX = ( int(grid_square_parameters.x_location / 7.8) if grid_square_parameters.x_location @@ -296,7 +297,7 @@ def do_insert_foil_hole( with Session() as db: db.add(record) db.commit() - log.info(f"Created FoilHole {record.gridSquareId}") + log.info(f"Created FoilHole {record.foilHoleId}") return {"success": True, "return_value": record.foilHoleId} except ispyb.ISPyBException as e: log.error( From 3a3c4c614e255861b74e454c852bc8b9997dec37 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 7 Feb 2025 09:43:05 +0000 Subject: [PATCH 3/5] pixel location for grid squares got repeated --- src/murfey/server/ispyb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index 6b6aa8ed2..aad927a4d 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -225,7 +225,6 @@ def do_update_grid_square( if grid_square_parameters.y_location else None ) - grid_square.pixelLocationY = grid_square_parameters.y_location grid_square.height = ( int(grid_square_parameters.height / 7.8) if grid_square_parameters.height is not None From 1555602af903dd5ad7f91ac801e07e101e62957c Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 7 Feb 2025 10:00:53 +0000 Subject: [PATCH 4/5] Attempt to fix the atlas path inserted into ispyb --- src/murfey/client/contexts/spa_metadata.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index db4b01547..e5a19d56a 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -66,11 +66,13 @@ def _atlas_destination( instrument_name=environment.instrument_name, demo=environment.demo, ) - if environment.visit in environment.default_destinations[source]: - return ( - Path(machine_config.get("rsync_basepath", "")) - / Path(environment.default_destinations[source]).parent - ) + for i, destination_part in enumerate( + Path(environment.default_destinations[source]).parts + ): + if destination_part == environment.visit: + return Path(machine_config.get("rsync_basepath", "")) / "/".join( + Path(environment.default_destinations[source]).parent.parts[: i + 1] + ) return ( Path(machine_config.get("rsync_basepath", "")) / Path(environment.default_destinations[source]).parent @@ -135,7 +137,6 @@ def post_transfer( atlas_pixel_size = atlas_original_pixel_size * 7.8 logger.info(f"Atlas image pixel size determined to be {atlas_pixel_size}") - sample = None for p in partial_path.split("/"): if p.startswith("Sample"): sample = int(p.replace("Sample", "")) @@ -167,7 +168,8 @@ def post_transfer( "tag": dcg_tag, "atlas": str( _atlas_destination(environment, source, transferred_file) - / environment.samples[source].atlas + / environment.samples[source].atlas.parent + / atlas_xml_path.with_suffix(".jpg").name ), "sample": environment.samples[source].sample, "atlas_pixel_size": atlas_pixel_size, From a7399cff641bb1763ee324a00a638d2223f15be4 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 7 Feb 2025 13:25:40 +0000 Subject: [PATCH 5/5] All of these should only be updated if a value is given --- src/murfey/server/ispyb.py | 78 ++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index aad927a4d..bfa086cac 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -215,30 +215,26 @@ def do_update_grid_square( ) if grid_square_parameters.image: grid_square.gridSquareImage = grid_square_parameters.image - grid_square.pixelLocationX = ( - int(grid_square_parameters.x_location / 7.8) - if grid_square_parameters.x_location - else None - ) - grid_square.pixelLocationY = ( - int(grid_square_parameters.y_location / 7.8) - if grid_square_parameters.y_location - else None - ) - grid_square.height = ( - int(grid_square_parameters.height / 7.8) - if grid_square_parameters.height is not None - else None - ) - grid_square.width = ( - int(grid_square_parameters.width / 7.8) - if grid_square_parameters.width is not None - else None - ) - grid_square.angle = grid_square_parameters.angle - grid_square.stageLocationX = grid_square_parameters.x_stage_position - grid_square.stageLocationY = grid_square_parameters.y_stage_position - grid_square.pixelSize = grid_square_parameters.pixel_size + if grid_square_parameters.x_location: + grid_square.pixelLocationX = int( + grid_square_parameters.x_location / 7.8 + ) + if grid_square_parameters.y_location: + grid_square.pixelLocationY = int( + grid_square_parameters.y_location / 7.8 + ) + if grid_square_parameters.height is not None: + grid_square.height = int(grid_square_parameters.height / 7.8) + if grid_square_parameters.width is not None: + grid_square.width = int(grid_square_parameters.width / 7.8) + if grid_square_parameters.angle: + grid_square.angle = grid_square_parameters.angle + if grid_square_parameters.x_stage_position: + grid_square.stageLocationX = grid_square_parameters.x_stage_position + if grid_square_parameters.y_stage_position: + grid_square.stageLocationY = grid_square_parameters.y_stage_position + if grid_square_parameters.pixel_size: + grid_square.pixelSize = grid_square_parameters.pixel_size db.add(grid_square) db.commit() return {"success": True, "return_value": grid_square.gridSquareId} @@ -317,24 +313,22 @@ def do_update_foil_hole( foil_hole = ( db.query(FoilHole).filter(FoilHole.foilHoleId == foil_hole_id).one() ) - foil_hole.foilHoleImage = foil_hole_parameters.image - foil_hole.pixelLocationX = ( - int(foil_hole_parameters.x_location * scale_factor) - if foil_hole_parameters.x_location - else None - ) - foil_hole.pixelLocationY = ( - int(foil_hole_parameters.y_location * scale_factor) - if foil_hole_parameters.y_location - else None - ) - foil_hole.diameter = ( - foil_hole_parameters.diameter * scale_factor - if foil_hole_parameters.diameter is not None - else None - ) - foil_hole.stageLocationX = foil_hole_parameters.x_stage_position - foil_hole.stageLocationY = foil_hole_parameters.y_stage_position + if foil_hole_parameters.image: + foil_hole.foilHoleImage = foil_hole_parameters.image + if foil_hole_parameters.x_location: + foil_hole.pixelLocationX = int( + foil_hole_parameters.x_location * scale_factor + ) + if foil_hole_parameters.y_location: + foil_hole.pixelLocationY = int( + foil_hole_parameters.y_location * scale_factor + ) + if foil_hole_parameters.diameter is not None: + foil_hole.diameter = foil_hole_parameters.diameter * scale_factor + if foil_hole_parameters.x_stage_position: + foil_hole.stageLocationX = foil_hole_parameters.x_stage_position + if foil_hole_parameters.y_stage_position: + foil_hole.stageLocationY = foil_hole_parameters.y_stage_position if ( foil_hole_parameters.readout_area_x is not None and foil_hole_parameters.thumbnail_size_x is not None