Skip to content

Multiplex file structure change #386

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
amyjaynethompson wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Multiplex file structure change #386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b0c3c44
update xia2.multiplex file upload for new file structure
amyjaynethompson Apr 30, 2026
873d87d
update dimple wrapper to put sym links to multiplex in parent folder …
amyjaynethompson Apr 30, 2026
7d9339b
update for new mplx file structure
amyjaynethompson Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/dlstbx/wrapper/dimple.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,8 +317,14 @@ def run(self):
os.fspath(self.results_directory), symlink
)
mtzsymlink = mtz.parent / symlink
if not mtzsymlink.exists():
if "multiplex" in mtzsymlink.name:
parts = list(mtzsymlink.parts)
parts.remove("DataFiles")
mtzsymlink = pathlib.Path(*parts)
deltapath = os.path.relpath(self.results_directory, mtz.parent.parent)
else:
deltapath = os.path.relpath(self.results_directory, mtz.parent)
if not mtzsymlink.exists():
os.symlink(deltapath, mtzsymlink)
for f in self.working_directory.iterdir():
if f.name.startswith("."):
Expand Down
169 changes: 132 additions & 37 deletions src/dlstbx/wrapper/xia2_multiplex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,15 +248,19 @@ def run(self):
self.log.debug(result.stderr)
self.log.info(f"working_directory: {working_directory}")

scaled_unmerged_mtz = working_directory / "scaled_unmerged.mtz"
multiplex_json = working_directory / "xia2.multiplex.json"
scaled_unmerged_mtz = working_directory / "DataFiles" / "scaled_unmerged.mtz"
multiplex_json = working_directory / "Processing" / "xia2.multiplex.json"

# Placeholder logic to keep existing functionality - TODO - review if this is needed still
if not (scaled_unmerged_mtz.is_file() and multiplex_json.is_file()):
success = False

# Create results directory
results_directory.mkdir(parents=True, exist_ok=True)
subdirs = ("DataFiles", "LogFiles", "Processing")
for subdir in subdirs:
new_dir = results_directory / subdir
new_dir.mkdir(parents=True, exist_ok=True)
if params.get("create_symlink"):
dlstbx.util.symlink.create_parent_symlink(
results_directory, params["create_symlink"]
Expand All @@ -275,6 +279,10 @@ def is_final_result(final_file: pathlib.Path) -> bool:
for patt in pipeline_final_params["patterns"]
)

for subdir in subdirs:
new_dir = final_directory / subdir
new_dir.mkdir(parents=True, exist_ok=True)

keep_ext = {
".png": None,
".log": "log",
Expand All @@ -285,6 +293,8 @@ def is_final_result(final_file: pathlib.Path) -> bool:
".mtz": None,
".html": "log",
".sca": None,
".bib": "misc", # want these copied, don't need in synchweb
".phil": "misc",
}
keep = {
"scaled.mtz": "result",
Expand All @@ -311,33 +321,57 @@ def is_final_result(final_file: pathlib.Path) -> bool:
d = json.load(fh)

for dataset_name, dataset in d["datasets"].items():
base_dir_logs = working_directory / "LogFiles"
base_dir_results = working_directory / "DataFiles"

if dataset_name == "All data":
base_dir = working_directory
base_dir_processing = working_directory / "Processing"
dimple_symlink = "dimple-xia2.multiplex"
cluster_prefix = ""
cluster_num = None
cluster_results = None
elif "coordinate cluster" in dataset_name:
cluster_count += 1
cluster_num = dataset_name.split(" ")[-1]
cluster_prefix = f"coordinate_cluster_{cluster_num}_"
base_dir = working_directory / f"coordinate_cluster_{cluster_num}"
base_dir_processing = (
working_directory
/ "Processing"
/ "coordinate_cluster_{cluster_num}"
)
dimple_symlink = (
f"dimple-xia2.multiplex-coordinate_cluster_{cluster_num}"
)
cluster_results = (
results_directory
/ "Processing"
/ "coordinate_cluster_{cluster_num}"
)
cluster_results.mkdir(parents=True, exist_ok=True)
cluster_final = (
final_directory
/ "Processing"
/ "coordinate_cluster_{cluster_num}"
)
cluster_final.mkdir(parents=True, exist_ok=True)
else:
self.log.warning(
f"Ignoring unrecognised dataset pattern {dataset_name}"
)
continue

scaled_unmerged_mtz = base_dir / f"{cluster_prefix}scaled_unmerged.mtz"
scaled_unmerged_mtz = (
base_dir_results / f"{cluster_prefix}scaled_unmerged.mtz"
)
i_obs = iotbx.merging_statistics.select_data(
scaled_unmerged_mtz.as_posix(), data_labels=None
)

merging_stats = dataset["merging_stats"]
merging_stats_anom = dataset["merging_stats_anom"]
with (base_dir / f"{cluster_prefix}merging-stats.json").open("w") as fh:
with (working_directory / f"{cluster_prefix}merging-stats.json").open(
"w"
) as fh:
json.dump(merging_stats, fh)

ispyb_d = {
Expand Down Expand Up @@ -379,7 +413,13 @@ def is_final_result(final_file: pathlib.Path) -> bool:
xtriage_results = dataset.get("xtriage")
attachments = []

for filename in set(primary_log_files + list(base_dir.iterdir())):
for filename in set(
primary_log_files
+ list(base_dir_processing.iterdir())
+ list(base_dir_results.iterdir())
+ list(base_dir_logs.iterdir())
+ list(working_directory.iterdir())
):
filetype = None
if not filename.is_file():
continue # primary log files may not actually exist
Expand All @@ -390,59 +430,114 @@ def is_final_result(final_file: pathlib.Path) -> bool:
if filetype is None:
continue

destination = results_directory / filename.name
if (
destination.as_posix() in allfiles
and filename not in primary_log_files
):
destination = (
results_directory / f"{cluster_prefix}{filename.name}"
)
# Work out if in Processing, DataFiles or LogFiles (parent_dir)

parent_dir = None

for i in subdirs:
if i in filename.parts:
parent_dir = i

if not parent_dir:
destination = results_directory / filename.name
else:
# Get folder of clustering results (subdir of Processing)

if cluster_results and parent_dir == "Processing":
if cluster_results.name in filename.parts:
destination = cluster_results / filename.name
else:
self.log.warning(f"Broken paths for {filename}")
continue
else:
destination = results_directory / parent_dir / filename.name

if destination.as_posix() not in allfiles:
self.log.debug(f"Copying {filename} to {destination}")
shutil.copy(filename, destination)
allfiles.append(destination.as_posix())

if pipeline_final_params and is_final_result(destination):
destination = final_directory / destination.name
if any(i in destination.parts for i in subdirs):
destination = (
final_directory
/ destination.parent.name
/ destination.name
)
else:
destination = final_directory / destination.name
if destination not in allfiles:
self.log.debug(f"Copying {filename} to {destination}")
shutil.copy(filename, destination)
allfiles.append(destination.as_posix())

# Files uploaded separately for each cluster
if filetype:
attachments.append(
{
"file_path": destination.parent.as_posix(),
"file_name": destination.name,
"file_type": filetype,
"importance_rank": (
1
if destination.name.endswith(
(
"scaled.mtz",
"xia2.multiplex.html",
"xia2.multiplex.log",
)
)
else 2
),
}
)

file_data = {
"file_path": destination.parent.as_posix(),
"file_name": destination.name,
"file_type": filetype,
"importance_rank": (
1
if destination.name.endswith(
(
"scaled.mtz",
"xia2.multiplex.html",
"xia2.multiplex.log",
)
)
else 2
),
}

if "DataFiles" in destination.parts and filetype:
# if it is a cluster, only append cluster files
if cluster_results and "coordinate_cluster" in destination.name:
attachments.append(file_data)
# if it is not a cluster, only append non-cluster files
elif (
not cluster_results
and "coordinate_cluster" not in destination.name
):
attachments.append(file_data)

elif "LogFiles" in destination.parts and filetype:
# attach all log files for clusters and non-clusters
attachments.append(file_data)

elif "Processing" in destination.parts and filetype:
# If cluster, only attach files that are in a cluster folder
if (
cluster_results
and cluster_results.name in destination.parts
):
attachments.append(file_data)
# it not a cluster, make sure file is not in a cluster folder
elif (
not cluster_results
and destination.parent.name == "Processing"
):
attachments.append(file_data)
else:
# Anything in the parent folder (ie main logs) should be attached for clusters and non-clusters
attachments.append(file_data)

# Add parameters to the environment to be picked up downstream by trigger function
self.recwrap.environment.update(
{
"scaled_mtz": (
results_directory / f"{cluster_prefix}scaled.mtz"
results_directory
/ "DataFiles"
/ "{cluster_prefix}scaled.mtz"
).as_posix()
}
)
self.recwrap.environment.update(
{
"scaled_unmerged_mtz": (
results_directory / f"{cluster_prefix}scaled_unmerged.mtz"
results_directory
/ "DataFiles"
/ "{cluster_prefix}scaled_unmerged.mtz"
).as_posix()
}
)
Expand Down
Loading