Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions conf/train.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ defaults:

config_version: 2

paths:
eozarr_root: null
preload_cache_root: null

# --- derived keys (resolved lazily by OmegaConf) ---
_spatial: mtg_new
_spatial_val: eval_mtg_new_val
Expand Down
9 changes: 6 additions & 3 deletions deep_lst/dataset/data_module.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@


class LitDataModule(pl.LightningDataModule):
def __init__(self, **kwargs):
def __init__(self, experiment_paths=None, **kwargs):
super().__init__()
self.kwargs = kwargs
self.paths = experiment_paths
self.collate_fn = Collator(kwargs["model"]["name"], get_packing_keys(kwargs["train_dataset"]))

def setup(self, stage=None):
Expand All @@ -27,9 +28,11 @@ def _build_split(self, split_kwargs):
modality_ids = self.kwargs["modality_ids"]
spatial = split_kwargs["spatial"]
if spatial not in self._shared_by_spatial:
self._shared_by_spatial[spatial] = MultimodalDataset.preload_shared(modality_ids, **split_kwargs)
self._shared_by_spatial[spatial] = MultimodalDataset.preload_shared(
modality_ids, paths=self.paths, **split_kwargs
)
shared = self._shared_by_spatial[spatial]
return MultimodalDataset.from_modalities(modality_ids, shared, **split_kwargs)
return MultimodalDataset.from_modalities(modality_ids, shared, paths=self.paths, **split_kwargs)

def train_dataloader(self):
return DataLoader(
Expand Down
25 changes: 14 additions & 11 deletions deep_lst/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,11 @@ def load_samples(
return indices


def _load_anchor_ids(spatial_name):
def _load_anchor_ids(spatial_name, paths=None):
"""Anchor cell ids for a split: global MTG ids for training, seviri equiangular pixels
for an eval holdout region."""
return np.sort(np.load(PATHS.spatial_indices_path(spatial_name))["src_indices"].astype(np.int64))
paths = paths or PATHS
return np.sort(np.load(paths.spatial_indices_path(spatial_name))["src_indices"].astype(np.int64))


def _flatten_hours(values, valid):
Expand Down Expand Up @@ -373,17 +374,18 @@ def non_land_mask(self, classes=(17, 18)) -> np.ndarray:
return np.isin(landcov[self.inverse[_ANCHOR_MODALITY]], classes)

@classmethod
def preload_shared(cls, modality_ids, **kwargs):
def preload_shared(cls, modality_ids, paths=None, **kwargs):
"""Preload each modality's native columns once, shared across train/val/test splits.

Every anchor cell (MTG for training, seviri pixels for eval) is nearest-neighbour mapped
to each modality's native store row; coarse modalities are deduped so only unique cells
are read. Columns depend only on (anchors, store), not the temporal split, so this runs
once — a full-axis cache per split would not fit in RAM.
"""
paths = paths or PATHS
spatial_name = kwargs["spatial"]
stats_tag = stats_spatial(kwargs)
anchor_ids = _load_anchor_ids(spatial_name)
anchor_ids = _load_anchor_ids(spatial_name, paths)
src_grid = anchor_grid(kwargs.get("anchor", "mtg"))
num_workers = kwargs.get("preload_workers", 8)

Expand All @@ -397,7 +399,7 @@ def preload_shared(cls, modality_ids, **kwargs):
modality: {
"store": kwargs[modality]["store"],
"load": kwargs[modality]["load"],
"stats_mtime": os.path.getmtime(PATHS.stats_zarr_path(modality, stats_tag) / "zarr.json"),
"stats_mtime": os.path.getmtime(paths.stats_zarr_path(modality, stats_tag) / "zarr.json"),
"axis": [PRELOAD_START_DAY, PRELOAD_END_DAY],
}
for modality in modality_ids
Expand All @@ -408,7 +410,7 @@ def preload_shared(cls, modality_ids, **kwargs):
if transform is not None: # only when set, so caches built before the key existed stay valid
fingerprint_spec["modalities"][modality]["transform"] = transform
fingerprint = hashlib.sha256(json.dumps(fingerprint_spec, sort_keys=True).encode()).hexdigest()[:16]
cache_path = PATHS.preload_cache_path(fingerprint, spatial_name)
cache_path = paths.preload_cache_path(fingerprint, spatial_name)
if cache_path.exists():
print(f"Loading preload cache {cache_path}")
return torch.load(cache_path, map_location="cpu", weights_only=False)
Expand All @@ -431,11 +433,11 @@ def preload_shared(cls, modality_ids, **kwargs):
accepted = inspect.signature(get_data_reader_class(modality)).parameters
reader_kwargs = {key: value for key, value in reader_kwargs.items() if key in accepted}
reader = eodata.get_data_reader(
modality, store=str(PATHS.eozarr_store_path(kwargs[modality]["store"])), **reader_kwargs
modality, store=str(paths.eozarr_store_path(kwargs[modality]["store"])), **reader_kwargs
)

normalizer = ModalityNormalizer.from_spec(
f"zscore:{PATHS.stats_zarr_path(modality, stats_tag)}", reader.schema
f"zscore:{paths.stats_zarr_path(modality, stats_tag)}", reader.schema
)
normalizer = with_transform(normalizer, kwargs[modality].get("transform"))
# Channel indices of the config variables within each data type's full store variable order.
Expand All @@ -462,17 +464,18 @@ def preload_shared(cls, modality_ids, **kwargs):
return shared

@classmethod
def from_modalities(cls, modality_ids, shared, **kwargs):
def from_modalities(cls, modality_ids, shared, paths=None, **kwargs):
"""Build one split's dataset from the shared preloaded columns + this split's anchors."""
paths = paths or PATHS
stats_tag = stats_spatial(kwargs)
computed_stats = {m: load_computed_stats(PATHS.stats_zarr_path(m, stats_tag)) for m in modality_ids}
computed_stats = {m: load_computed_stats(paths.stats_zarr_path(m, stats_tag)) for m in modality_ids}

if kwargs["spatial"] != shared["spatial"]:
raise ValueError(f"Shared cache built for spatial={shared['spatial']!r}, got {kwargs['spatial']!r}")

temporal_split = kwargs["temporal"]
if temporal_split is not None:
temporal_paths = PATHS.temporal_indices_path(temporal_split)
temporal_paths = paths.temporal_indices_path(temporal_split)
else:
temporal_paths = None

Expand Down
5 changes: 2 additions & 3 deletions deep_lst/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,10 @@ def _git_hash() -> str:

@hydra.main(config_path="../conf", config_name="train", version_base="1.3")
def main(cfg: DictConfig) -> None:
paths = ExperimentPaths()

config = OmegaConf.to_container(cfg, resolve=True)
choices = HydraConfig.get().runtime.choices
validate_config(config)
paths = ExperimentPaths(**{key: Path(value) for key, value in config.get("paths", {}).items() if value is not None})
pl.seed_everything(config["seed"], workers=True)

smoke = config.pop("smoke", False)
Expand Down Expand Up @@ -156,7 +155,7 @@ def main(cfg: DictConfig) -> None:
check_stats_transform(paths.stats_zarr_path(modality, stats_tag), transform)

data_module = LitDataModule(
**config, train_dataset=train_view, val_dataset=views["val"], test_dataset=views["test"]
experiment_paths=paths, **config, train_dataset=train_view, val_dataset=views["val"], test_dataset=views["test"]
)
data_module.setup()

Expand Down
9 changes: 8 additions & 1 deletion deep_lst/utils/io.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import getpass
import os
import re
from dataclasses import dataclass, field
from pathlib import Path
Expand All @@ -14,7 +16,12 @@ class ExperimentPaths:
eozarr_root: Path = field(
default_factory=lambda: Path("/Net/Groups/BGI/work_5/scratch/WeatherGenerator/data/lowres_old")
)
preload_cache_root: Path = field(default_factory=lambda: Path("/Net/Groups/BGI/tscratch/mwessel/tmp"))
preload_cache_root: Path = field(
default_factory=lambda: Path(
os.environ.get("DEEP_LST_CACHE_ROOT")
or f"/Net/Groups/BGI/tscratch/{os.environ.get('USER') or getpass.getuser()}/tmp"
)
)

def experiment_dir(self, experiment: str) -> Path:
return self.experiment_root / experiment
Expand Down
22 changes: 22 additions & 0 deletions docs/CONFIGURATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,28 @@ be edited.

## 2. `conf/train.yaml`

### Training data and cache paths

`paths.eozarr_root` and `paths.preload_cache_root` override the data-store and preload-cache
roots used by `deep_lst/train.py`, for all three training/validation/test splits. The defaults
are defined in `ExperimentPaths` in `deep_lst/utils/io.py`, also used by the `PATHS` singleton
in `deep_lst/dataset/dataset.py`. Both config values default to `null`, so these defaults apply.

The cache default uses `DEEP_LST_CACHE_ROOT` when it is nonempty; otherwise it uses
`/Net/Groups/BGI/tscratch/$USER/tmp` (the login name is used if `USER` is unset or empty).
An explicit `paths.preload_cache_root` takes priority over this default. Relative paths remain
relative to the repository working directory. The overrides are saved in the run's `config.yml`.

```
uv run python deep_lst/train.py paths.eozarr_root=/data/eozarr paths.preload_cache_root=/scratch/deep_lst-cache
```

Cache keys are unchanged; use a separate cache root when the source data content differs.

These Hydra overrides apply to training. Standalone inference, statistics and fine-tuning use

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good for this self-contained addition. But this override should eventually be applied to all modules, I'll add it to the issue.
React

the shared `ExperimentPaths` defaults, including the cache default described above.
Model, loss and data-processing settings are unchanged.

### 2.1 Selections and splits

| Key | Default | Meaning |
Expand Down
12 changes: 8 additions & 4 deletions docs/RUNBOOK.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,14 @@ assets/statistics/<modality>_stats_<name>.zarr z-score statistics per (modalit
assets/baselines/<kind>/<name>.nc persistence / climatology / ERA5(-Land) reference forecasts
```

Two hard-coded paths need attention in a new clone:

1. `ExperimentPaths.preload_cache_root` in `deep_lst/utils/io.py` points at the maintainer's
tscratch directory, which you cannot write to. Point it at your own tscratch directory.
Check these locations in a new clone:

1. `ExperimentPaths.preload_cache_root` in `deep_lst/utils/io.py` uses `DEEP_LST_CACHE_ROOT`
when it is nonempty, otherwise `/Net/Groups/BGI/tscratch/$USER/tmp` (using the login name if
`USER` is unset or empty). For training, `paths.preload_cache_root=<your-tscratch>` on the command
line or in the config file takes priority; `paths.eozarr_root` can also override the store
root. Other entry points use the shared `ExperimentPaths` defaults, including this cache
fallback. See `docs/CONFIGURATION.md`.
The data stores (`ExperimentPaths.eozarr_root`,
`/Net/Groups/BGI/work_5/scratch/WeatherGenerator/data/lowres_old`) are group-readable and
stay as they are.
Expand Down
12 changes: 11 additions & 1 deletion tests/test_config_golden.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@
"_sigma_prognostics",
"_sigma_diagnostics",
]
NEW_TOP_LEVEL = ["config_version", "objective", "optim", "window", "sources", "features", "splits"]
NEW_TOP_LEVEL = ["config_version", "paths", "objective", "optim", "window", "sources", "features", "splits"]
# the sections migrate_config has to rebuild exactly
MIGRATED_SECTIONS = [
"config_version",
Expand Down Expand Up @@ -203,6 +203,16 @@ def test_every_other_key_is_unchanged(case):
assert sorted(new) == sorted(untouched + ["model"] + NEW_TOP_LEVEL)


def test_training_path_overrides_leave_scientific_config_unchanged():
default = _compose(NEW_OVERRIDES["default"])
overridden = _compose(
[*NEW_OVERRIDES["default"], "paths.eozarr_root=local-data", "paths.preload_cache_root=local-cache"]
)
assert default.pop("paths") == {"eozarr_root": None, "preload_cache_root": None}
assert overridden.pop("paths") == {"eozarr_root": "local-data", "preload_cache_root": "local-cache"}
assert overridden == default


@pytest.mark.parametrize("case", CASES)
def test_migrated_fixture_equals_the_new_composition(case):
migrated = config_utils.migrate_config(_fixture(case))
Expand Down
31 changes: 31 additions & 0 deletions tests/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,37 @@ def paths(tmp_path):
)


def test_cache_root_uses_environment_without_user(monkeypatch, tmp_path):
monkeypatch.setenv("DEEP_LST_CACHE_ROOT", str(tmp_path / "cache"))
monkeypatch.delenv("USER", raising=False)
assert ExperimentPaths().preload_cache_root == tmp_path / "cache"


@pytest.mark.parametrize("cache_root", [None, ""])
def test_cache_root_defaults_to_current_user(monkeypatch, cache_root):
if cache_root is None:
monkeypatch.delenv("DEEP_LST_CACHE_ROOT", raising=False)
else:
monkeypatch.setenv("DEEP_LST_CACHE_ROOT", cache_root)
monkeypatch.setenv("USER", "test-user")
assert ExperimentPaths().preload_cache_root.as_posix() == "/Net/Groups/BGI/tscratch/test-user/tmp"


def test_cache_root_uses_account_name_when_user_is_unset(monkeypatch):
monkeypatch.delenv("DEEP_LST_CACHE_ROOT", raising=False)
monkeypatch.delenv("USER", raising=False)
monkeypatch.setattr("getpass.getuser", lambda: "test-user")
assert ExperimentPaths().preload_cache_root.as_posix() == "/Net/Groups/BGI/tscratch/test-user/tmp"


def test_explicit_cache_root_overrides_environment(monkeypatch, tmp_path):
monkeypatch.setenv("DEEP_LST_CACHE_ROOT", str(tmp_path / "environment-cache"))
assert (
ExperimentPaths(preload_cache_root=tmp_path / "explicit-cache").preload_cache_root
== tmp_path / "explicit-cache"
)


# ── directory resolution ──────────────────────────────────────────────────────


Expand Down
Loading
Loading