Skip to content

Commit 840fb30

Browse files
stephen-riggsstephen-riggs
authored
Remove the old SPA context which we no longer use (#453)
1 parent 7c2a7d9 commit 840fb30

11 files changed

Lines changed: 17 additions & 380 deletions

File tree

‎src/murfey/client/analyser.py‎

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
from murfey.client.context import Context
1818
from murfey.client.contexts.clem import CLEMContext
19-
from murfey.client.contexts.spa import SPAContext, SPAModularContext
19+
from murfey.client.contexts.spa import SPAModularContext
2020
from murfey.client.contexts.spa_metadata import SPAMetadataContext
2121
from murfey.client.contexts.tomo import TomographyContext
2222
from murfey.client.instance_environment import MurfeyInstanceEnvironment
@@ -159,23 +159,7 @@ def _find_context(self, file_path: Path) -> bool:
159159
if split_file_name[0].startswith("FoilHole"):
160160
if not self._context:
161161
logger.info("Acquisition software: EPU")
162-
if self._environment:
163-
try:
164-
cfg = get_machine_config_client(
165-
str(self._environment.url.geturl()),
166-
instrument_name=self._environment.instrument_name,
167-
demo=self._environment.demo,
168-
)
169-
except Exception as e:
170-
logger.error(f"Exception encountered: {e}")
171-
cfg = {}
172-
else:
173-
cfg = {}
174-
self._context = (
175-
SPAModularContext("epu", self._basepath)
176-
if cfg.get("modular_spa")
177-
else SPAContext("epu", self._basepath)
178-
)
162+
self._context = SPAModularContext("epu", self._basepath)
179163
self.parameters_model = ProcessingParametersSPA
180164
return True
181165

@@ -325,8 +309,7 @@ def _analyse(self):
325309
"form": dc_metadata,
326310
"dependencies": (
327311
spa_form_dependencies
328-
if isinstance(self._context, SPAContext)
329-
or isinstance(
312+
if isinstance(
330313
self._context, SPAModularContext
331314
)
332315
else {}
@@ -385,8 +368,7 @@ def _analyse(self):
385368
"form": dc_metadata,
386369
"dependencies": (
387370
spa_form_dependencies
388-
if isinstance(self._context, SPAContext)
389-
or isinstance(
371+
if isinstance(
390372
self._context, SPAModularContext
391373
)
392374
else {}

‎src/murfey/client/contexts/spa.py‎

Lines changed: 1 addition & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ def _get_xml_list_index(key: str, xml_list: list) -> int:
9292
raise ValueError(f"Key not found in XML list: {key}")
9393

9494

95-
class _SPAContext(Context):
95+
class SPAModularContext(Context):
9696
user_params = [
9797
ProcessingParameter(
9898
"dose_per_frame",
@@ -133,7 +133,6 @@ def __init__(self, acquisition_software: str, basepath: Path):
133133
super().__init__("SPA", acquisition_software)
134134
self._basepath = basepath
135135
self._processing_job_stash: dict = {}
136-
self._preprocessing_triggers: dict = {}
137136
self._foil_holes: Dict[int, List[int]] = {}
138137

139138
def gather_metadata(
@@ -365,8 +364,6 @@ def gather_metadata(
365364
) or True
366365
return metadata
367366

368-
369-
class SPAModularContext(_SPAContext):
370367
def _position_analysis(
371368
self,
372369
transferred_file: Path,
@@ -661,104 +658,3 @@ def _launch_spa_pipeline(
661658
url: str = "",
662659
):
663660
return
664-
665-
666-
class SPAContext(_SPAContext):
667-
def _register_data_collection(
668-
self,
669-
tag: str,
670-
url: str,
671-
data: dict,
672-
environment: MurfeyInstanceEnvironment,
673-
):
674-
logger.info(f"registering data collection with data {data}")
675-
environment.id_tag_registry["data_collection"].append(tag)
676-
image_directory = str(environment.default_destinations[Path(tag)])
677-
json = {
678-
"voltage": data["voltage"],
679-
"pixel_size_on_image": data["pixel_size_on_image"],
680-
"experiment_type": data["experiment_type"],
681-
"image_size_x": data["image_size_x"],
682-
"image_size_y": data["image_size_y"],
683-
"file_extension": data["file_extension"],
684-
"acquisition_software": data["acquisition_software"],
685-
"image_directory": image_directory,
686-
"tag": tag,
687-
"source": tag,
688-
"magnification": data["magnification"],
689-
"total_exposed_dose": data.get("total_exposed_dose"),
690-
"c2aperture": data.get("c2aperture"),
691-
"exposure_time": data.get("exposure_time"),
692-
"slit_width": data.get("slit_width"),
693-
"phase_plate": data.get("phase_plate", False),
694-
}
695-
capture_post(url, json=json)
696-
697-
def post_transfer(
698-
self,
699-
transferred_file: Path,
700-
environment: MurfeyInstanceEnvironment | None = None,
701-
**kwargs,
702-
) -> bool:
703-
return True
704-
705-
def _register_processing_job(
706-
self,
707-
tag: str,
708-
environment: MurfeyInstanceEnvironment,
709-
parameters: Dict[str, Any] | None = None,
710-
):
711-
logger.info(f"registering processing job with parameters: {parameters}")
712-
parameters = parameters or {}
713-
environment.id_tag_registry["processing_job"].append(tag)
714-
proc_url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_processing_job"
715-
machine_config = get_machine_config_client(
716-
str(environment.url.geturl()),
717-
instrument_name=environment.instrument_name,
718-
demo=environment.demo,
719-
)
720-
image_directory = str(
721-
Path(machine_config.get("rsync_basepath", "."))
722-
/ environment.default_destinations[Path(tag)]
723-
)
724-
if self._acquisition_software == "epu":
725-
import_images = f"{Path(image_directory).resolve()}/GridSquare*/Data/*{parameters['file_extension']}"
726-
else:
727-
import_images = (
728-
f"{Path(image_directory).resolve()}/*{parameters['file_extension']}"
729-
)
730-
msg: Dict[str, Any] = {
731-
"tag": tag,
732-
"source": tag,
733-
"recipe": "ispyb-relion",
734-
"parameters": {
735-
"acquisition_software": parameters["acquisition_software"],
736-
"voltage": parameters["voltage"],
737-
"gain_ref": parameters["gain_ref"],
738-
"dose_per_frame": parameters["dose_per_frame"],
739-
"eer_grouping": parameters["eer_fractionation"],
740-
"import_images": import_images,
741-
"angpix": float(parameters["pixel_size_on_image"]) * 1e10,
742-
"symmetry": parameters["symmetry"],
743-
"boxsize": parameters["boxsize"],
744-
"downscale": parameters["downscale"],
745-
"small_boxsize": parameters["small_boxsize"],
746-
"mask_diameter": parameters["mask_diameter"],
747-
"use_cryolo": parameters["use_cryolo"],
748-
"estimate_particle_diameter": parameters["estimate_particle_diameter"],
749-
},
750-
}
751-
if parameters["particle_diameter"]:
752-
msg["parameters"]["particle_diameter"] = parameters["particle_diameter"]
753-
capture_post(proc_url, json=msg)
754-
755-
def _launch_spa_pipeline(
756-
self,
757-
tag: str,
758-
jobid: int,
759-
environment: MurfeyInstanceEnvironment,
760-
url: str = "",
761-
):
762-
environment.id_tag_registry["auto_proc_program"].append(tag)
763-
data = {"job_id": jobid}
764-
capture_post(url, json=data)

‎src/murfey/client/contexts/tomo.py‎

Lines changed: 0 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
import requests
1010
import xmltodict
11-
from pydantic import BaseModel
1211

1312
import murfey.util.eer
1413
from murfey.client.context import Context, ProcessingParameter
@@ -17,7 +16,6 @@
1716
MovieTracker,
1817
MurfeyID,
1918
MurfeyInstanceEnvironment,
20-
global_env_lock,
2119
)
2220
from murfey.util import authorised_requests, capture_post, get_machine_config_client
2321
from murfey.util.mdoc import get_block, get_global_data, get_num_blocks
@@ -65,15 +63,6 @@ def _construct_tilt_series_name(file_path: Path) -> str:
6563
return "_".join(split_name[:-5])
6664

6765

68-
class ProcessFileIncomplete(BaseModel):
69-
dest: Path
70-
source: Path
71-
image_number: int
72-
mc_uuid: int
73-
tag: str
74-
description: str = ""
75-
76-
7766
class TomographyContext(Context):
7867
user_params = [
7968
ProcessingParameter(
@@ -101,7 +90,6 @@ def __init__(self, acquisition_software: str, basepath: Path):
10190
self._aligned_tilt_series: List[str] = []
10291
self._data_collection_stash: list = []
10392
self._processing_job_stash: dict = {}
104-
self._preprocessing_triggers: dict = {}
10593
self._lock: RLock = RLock()
10694

10795
def _flush_data_collections(self):
@@ -120,12 +108,6 @@ def _flush_data_collections(self):
120108
capture_post(dc_data[0], json=data)
121109
self._data_collection_stash = []
122110

123-
def _flush_processing_job(self, tag: str):
124-
if proc_data := self._processing_job_stash.get(tag):
125-
for pd in proc_data:
126-
requests.post(pd[0], json=pd[1])
127-
self._processing_job_stash.pop(tag)
128-
129111
def _flush_processing_jobs(self):
130112
logger.info(
131113
f"Flushing {len(self._processing_job_stash.keys())} processing job API calls"
@@ -135,75 +117,6 @@ def _flush_processing_jobs(self):
135117
requests.post(pd[0], json=pd[1])
136118
self._processing_job_stash = {}
137119

138-
def _flush_preprocess(self, tag: str, app_id: int):
139-
if tag_tr := self._preprocessing_triggers.get(tag):
140-
for tr in tag_tr:
141-
process_file = self._complete_process_file(tr[1], tr[2], app_id)
142-
if process_file:
143-
capture_post(tr[0], json=process_file)
144-
self._preprocessing_triggers.pop(tag)
145-
146-
def _complete_process_file(
147-
self,
148-
incomplete_process_file: ProcessFileIncomplete,
149-
environment: MurfeyInstanceEnvironment,
150-
app_id: int,
151-
) -> dict:
152-
try:
153-
with global_env_lock:
154-
tag = incomplete_process_file.tag
155-
156-
eer_fractionation_file = None
157-
if environment.data_collection_parameters.get("num_eer_frames"):
158-
response = requests.post(
159-
f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/eer_fractionation_file",
160-
json={
161-
"num_frames": environment.data_collection_parameters[
162-
"num_eer_frames"
163-
],
164-
"fractionation": environment.data_collection_parameters[
165-
"eer_fractionation"
166-
],
167-
"dose_per_frame": environment.data_collection_parameters[
168-
"dose_per_frame"
169-
],
170-
"fractionation_file_name": "eer_fractionation_tomo.txt",
171-
},
172-
)
173-
eer_fractionation_file = response.json()["eer_fractionation_file"]
174-
175-
new_dict = {
176-
"path": str(incomplete_process_file.dest),
177-
"description": incomplete_process_file.description,
178-
"size": incomplete_process_file.source.stat().st_size,
179-
"timestamp": incomplete_process_file.source.stat().st_ctime,
180-
"processing_job": environment.processing_job_ids[tag][
181-
"em-tomo-preprocess"
182-
],
183-
"data_collection_id": environment.data_collection_ids[tag],
184-
"image_number": incomplete_process_file.image_number,
185-
"pixel_size": environment.data_collection_parameters[
186-
"pixel_size_on_image"
187-
],
188-
"autoproc_program_id": app_id,
189-
"mc_uuid": incomplete_process_file.mc_uuid,
190-
"dose_per_frame": environment.data_collection_parameters.get(
191-
"dose_per_frame"
192-
),
193-
"mc_binning": environment.data_collection_parameters.get(
194-
"motion_corr_binning", 1
195-
),
196-
"gain_ref": environment.data_collection_parameters.get("gain_ref"),
197-
"voltage": environment.data_collection_parameters.get(
198-
"voltage", 300
199-
),
200-
"eer_fractionation_file": eer_fractionation_file,
201-
}
202-
return new_dict
203-
except KeyError:
204-
logger.warning("Key error encountered in _complete_process_file")
205-
return {}
206-
207120
def _file_transferred_to(
208121
self, environment: MurfeyInstanceEnvironment, source: Path, file_path: Path
209122
):
@@ -441,14 +354,10 @@ def _add_tilt(
441354
preproc_data = {
442355
"path": str(file_transferred_to),
443356
"description": "",
444-
"data_collection_id": environment.data_collection_ids.get(tilt_series),
445357
"image_number": environment.movies[file_transferred_to].movie_number,
446358
"pixel_size": environment.data_collection_parameters.get(
447359
"pixel_size_on_image", 0
448360
),
449-
"autoproc_program_id": environment.autoproc_program_ids.get(
450-
tilt_series, {}
451-
).get("em-tomo-preprocess"),
452361
"dose_per_frame": environment.data_collection_parameters.get(
453362
"dose_per_frame", 0
454363
),

0 commit comments

Comments
 (0)