From e84eb2419d38a3a0339f179fc809868668d649d0 Mon Sep 17 00:00:00 2001 From: ramin_yz Date: Fri, 13 Nov 2026 21:35:57 +0100 Subject: [PATCH 1/2] feat: allow a separate store root for each modality Read optional sources..store_root through the existing path resolver. Forward it to dataset, statistics and baseline metadata readers. Keep the shared-root fallback and directory/ZIP selection unchanged. Include explicit modality roots in preload cache keys so changing roots cannot return data cached from another location. Keep existing default cache keys and scientific settings unchanged. Add focused path, config, cache and reader-forwarding regressions. --- deep_lst/baselines/base.py | 5 +- deep_lst/dataset/dataset.py | 14 ++++- deep_lst/utils/io.py | 9 +-- scripts/assets/compute_statistics.py | 1 + tests/test_config_golden.py | 11 ++++ tests/test_io.py | 22 +++++++ tests/test_reader_store_roots.py | 89 ++++++++++++++++++++++++++++ tests/test_training_paths.py | 25 +++++++- 8 files changed, 165 insertions(+), 11 deletions(-) create mode 100644 tests/test_reader_store_roots.py diff --git a/deep_lst/baselines/base.py b/deep_lst/baselines/base.py index fa6f863..9ac0209 100644 --- a/deep_lst/baselines/base.py +++ b/deep_lst/baselines/base.py @@ -86,7 +86,10 @@ def _preload_series(self, shared: dict, modality: str, var_name: str): def _metadata_attrs(self, modality: str, store_name: str, var_name: str) -> dict: """Baseline attrs sourced from the store's own per-variable metadata scheme.""" - reader = get_data_reader(modality, store=str(self.paths.eozarr_store_path(store_name)), average_hourly=False) + store_root = self.config["test_dataset"][modality].get("store_root") + reader = get_data_reader( + modality, store=str(self.paths.eozarr_store_path(store_name, store_root)), average_hourly=False + ) variable_meta = reader.data_arr.attrs.get("metadata", {}).get(var_name, {}) return dict( variable=var_name, diff --git a/deep_lst/dataset/dataset.py b/deep_lst/dataset/dataset.py index bc96dce..1a38beb 100644 --- a/deep_lst/dataset/dataset.py +++ b/deep_lst/dataset/dataset.py @@ -3,6 +3,7 @@ import json import os from concurrent.futures import ThreadPoolExecutor +from pathlib import Path import eodata import numpy as np @@ -242,7 +243,7 @@ def __getitem__(self, index): return leaves @classmethod - def from_config(cls, modality, store, spatial, anchor, temporal, workers=1): + def from_config(cls, modality, store, spatial, anchor, temporal, workers=1, store_root=None): """Stream the MTG pixel selection (spatial) over the statistics timestep subset (temporal).""" anchor_ids = _load_anchor_ids(spatial) rows = np.unique(map_to_store_rows(anchor_grid(anchor), anchor_ids, store_grid(modality))) @@ -260,7 +261,9 @@ def from_config(cls, modality, store, spatial, anchor, temporal, workers=1): } accepted = inspect.signature(get_data_reader_class(modality)).parameters reader_kwargs = {key: value for key, value in reader_kwargs.items() if key in accepted} - reader = eodata.get_data_reader(modality, store=str(PATHS.eozarr_store_path(store)), **reader_kwargs) + reader = eodata.get_data_reader( + modality, store=str(PATHS.eozarr_store_path(store, store_root)), **reader_kwargs + ) return cls(reader, rows, stat_days) @@ -406,6 +409,9 @@ def preload_shared(cls, modality_ids, paths=None, **kwargs): }, } for modality in modality_ids: + store_root = kwargs[modality].get("store_root") + if store_root is not None: + fingerprint_spec["modalities"][modality]["store_root"] = str(Path(store_root).resolve()) transform = kwargs[modality].get("transform") if transform is not None: # only when set, so caches built before the key existed stay valid fingerprint_spec["modalities"][modality]["transform"] = transform @@ -433,7 +439,9 @@ def preload_shared(cls, modality_ids, paths=None, **kwargs): accepted = inspect.signature(get_data_reader_class(modality)).parameters reader_kwargs = {key: value for key, value in reader_kwargs.items() if key in accepted} reader = eodata.get_data_reader( - modality, store=str(paths.eozarr_store_path(kwargs[modality]["store"])), **reader_kwargs + modality, + store=str(paths.eozarr_store_path(kwargs[modality]["store"], kwargs[modality].get("store_root"))), + **reader_kwargs, ) normalizer = ModalityNormalizer.from_spec( diff --git a/deep_lst/utils/io.py b/deep_lst/utils/io.py index 216a887..f809499 100644 --- a/deep_lst/utils/io.py +++ b/deep_lst/utils/io.py @@ -50,14 +50,15 @@ def config_path(self, experiment: str) -> Path: def statistics_path(self, modality, name) -> Path: return self.assets_root / modality / "statistics" / f"statistics_{name}.npz" - def eozarr_store_path(self, store_name: str) -> Path: + def eozarr_store_path(self, store_name: str, store_root: str | Path | None = None) -> Path: """Resolve a full eodata .eozarr store from its config-provided name. Prefers the directory store when it is readable and falls back to the .zip sibling: - under eozarr_root the era5, era5_land, glofas and swe stores exist only as zips, while + under the selected root the era5, era5_land, glofas and swe stores exist only as zips, while seviri is a directory store.""" - directory = self.eozarr_root / f"{store_name}.eozarr" - archive = self.eozarr_root / f"{store_name}.eozarr.zip" + root = self.eozarr_root if store_root is None else Path(store_root) + directory = root / f"{store_name}.eozarr" + archive = root / f"{store_name}.eozarr.zip" directory_is_readable = (directory / "data" / "zarr.json").exists() if not directory_is_readable and archive.exists(): return archive diff --git a/scripts/assets/compute_statistics.py b/scripts/assets/compute_statistics.py index 3637478..65c2c97 100644 --- a/scripts/assets/compute_statistics.py +++ b/scripts/assets/compute_statistics.py @@ -101,6 +101,7 @@ def main(cfg: DictConfig) -> None: dataset = UnimodalDataset.from_config( modality=modality, store=source["store"], + store_root=source.get("store_root"), spatial=config["spatial"], anchor=config["anchor"], temporal=config["temporal"], diff --git a/tests/test_config_golden.py b/tests/test_config_golden.py index e27d6be..d8e880c 100644 --- a/tests/test_config_golden.py +++ b/tests/test_config_golden.py @@ -213,6 +213,17 @@ def test_training_path_overrides_leave_scientific_config_unchanged(): assert overridden == default +def test_modality_store_root_reaches_each_split_without_changing_other_settings(): + default = _compose(NEW_OVERRIDES["default"]) + overridden = _compose([*NEW_OVERRIDES["default"], "+sources.seviri.store_root=local-seviri"]) + for view in config_utils.split_views(overridden).values(): + assert view["seviri"]["store_root"] == "local-seviri" + assert "store_root" not in view["era5"] + assert "store_root" not in view["era5_land"] + assert overridden["sources"]["seviri"].pop("store_root") == "local-seviri" + assert overridden == default + + @pytest.mark.parametrize("case", CASES) def test_migrated_fixture_equals_the_new_composition(case): migrated = config_utils.migrate_config(_fixture(case)) diff --git a/tests/test_io.py b/tests/test_io.py index 560776b..538591c 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -4,6 +4,8 @@ hermetic — nothing depends on the real on-disk layout. """ +from pathlib import Path + import pytest from deep_lst.utils.io import ExperimentPaths @@ -49,6 +51,26 @@ def test_explicit_cache_root_overrides_environment(monkeypatch, tmp_path): ) +@pytest.mark.parametrize("root_kind", ["default", "string", "path", "relative"]) +def test_store_root_override_keeps_directory_and_zip_selection(tmp_path, monkeypatch, root_kind): + monkeypatch.chdir(tmp_path) + paths = ExperimentPaths(eozarr_root=tmp_path / "shared") + root = paths.eozarr_root if root_kind == "default" else tmp_path / "modality" + if root_kind == "relative": + root = Path("modality") + kwargs = {} if root_kind == "default" else {"store_root": str(root) if root_kind == "string" else root} + directory = root / "tiny.eozarr" + archive = root / "tiny.eozarr.zip" + root.mkdir(parents=True) + assert paths.eozarr_store_path("tiny", **kwargs) == directory + archive.touch() + assert paths.eozarr_store_path("tiny", **kwargs) == archive + (directory / "data").mkdir(parents=True) + (directory / "data" / "zarr.json").write_text("{}") + assert paths.eozarr_store_path("tiny", **kwargs) == directory + assert paths.eozarr_root == tmp_path / "shared" + + # ── directory resolution ────────────────────────────────────────────────────── diff --git a/tests/test_reader_store_roots.py b/tests/test_reader_store_roots.py new file mode 100644 index 0000000..5e88f02 --- /dev/null +++ b/tests/test_reader_store_roots.py @@ -0,0 +1,89 @@ +"""Per-modality store roots for statistics and baseline metadata readers.""" + +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +from omegaconf import OmegaConf + +pytest.importorskip("eodata") +from deep_lst.baselines import base # noqa: E402 +from deep_lst.baselines.era5land import Era5LandExtractor # noqa: E402 +from deep_lst.dataset import dataset # noqa: E402 +from deep_lst.utils.io import ExperimentPaths # noqa: E402 +from scripts.assets import compute_statistics # noqa: E402 + + +@pytest.mark.parametrize("store_root", [None, "custom-store"]) +def test_statistics_dataset_uses_the_selected_store_root(tmp_path, monkeypatch, store_root): + monkeypatch.chdir(tmp_path) + paths = ExperimentPaths(assets_root=tmp_path / "assets", eozarr_root=tmp_path / "default-store") + for path, values in ( + (paths.spatial_indices_path("tiny"), {"src_indices": [0]}), + (paths.temporal_indices_path("tiny"), {"time_indices": [0, 24]}), + ): + path.parent.mkdir(parents=True, exist_ok=True) + np.savez(path, **values) + monkeypatch.setattr(dataset, "PATHS", paths) + monkeypatch.setattr(dataset, "anchor_grid", lambda _: None) + monkeypatch.setattr(dataset, "store_grid", lambda _: None) + monkeypatch.setattr(dataset, "map_to_store_rows", lambda *args: np.array([4])) + monkeypatch.setattr(dataset, "get_data_reader_class", lambda _: lambda store, average_hourly: None) + stores = [] + reader = SimpleNamespace() + + def open_reader(modality, store, **kwargs): + stores.append(store) + return reader + + monkeypatch.setattr(dataset.eodata, "get_data_reader", open_reader) + kwargs = {} if store_root is None else {"store_root": store_root} + result = dataset.UnimodalDataset.from_config("seviri", "tiny", "tiny", "mtg", "tiny", **kwargs) + expected_root = paths.eozarr_root if store_root is None else Path(store_root) + assert stores == [str(expected_root / "tiny.eozarr")] + assert result.reader is reader + np.testing.assert_array_equal(result.rows, [4]) + assert result.stat_days == [dataset.PRELOAD_START_DAY, dataset.PRELOAD_START_DAY + 1] + + +@pytest.mark.parametrize("store_root", [None, "custom-store"]) +def test_statistics_command_forwards_the_source_store_root(monkeypatch, store_root): + source = {"store": "tiny"} + if store_root is not None: + source["store_root"] = store_root + cfg = OmegaConf.create( + dict(modality="seviri", sources={"seviri": source}, spatial="tiny", anchor="mtg", temporal="tiny") + ) + + def stop_at_dataset(**kwargs): + assert kwargs["store"] == "tiny" + assert kwargs.get("store_root") == store_root + raise RuntimeError("statistics dataset reached") + + monkeypatch.setattr(compute_statistics.UnimodalDataset, "from_config", staticmethod(stop_at_dataset)) + with pytest.raises(RuntimeError, match="statistics dataset reached"): + compute_statistics.main.__wrapped__(cfg) + + +@pytest.mark.parametrize("store_root", [None, "custom-store"]) +def test_baseline_metadata_uses_the_modality_store_root(tmp_path, monkeypatch, store_root): + monkeypatch.chdir(tmp_path) + extractor = Era5LandExtractor("skt", 2024) + extractor.paths = ExperimentPaths(eozarr_root=tmp_path / "default-store") + source = {"store": "tiny"} + if store_root is not None: + source["store_root"] = store_root + extractor.config = {"test_dataset": {"era5_land": source}} + stores = [] + + def open_reader(modality, store, **kwargs): + assert modality == "era5_land" + stores.append(store) + return SimpleNamespace(data_arr=SimpleNamespace(attrs={"metadata": {"skt": {"units": "K"}}})) + + monkeypatch.setattr(base, "get_data_reader", open_reader) + attrs = extractor._metadata_attrs("era5_land", "tiny", "skt") + expected_root = extractor.paths.eozarr_root if store_root is None else Path(store_root) + assert stores == [str(expected_root / "tiny.eozarr")] + assert attrs == dict(variable="skt", units="K", long_name="skt", description="", test_year="2024") diff --git a/tests/test_training_paths.py b/tests/test_training_paths.py index 9e2e052..f96c6ce 100644 --- a/tests/test_training_paths.py +++ b/tests/test_training_paths.py @@ -84,7 +84,7 @@ def test_configured_reader_and_cache_preserve_samples_on_cold_and_warm_load(tmp_ def reader(modality, store, **kwargs): stores.append(store) - return SimpleNamespace(schema={}, data_arr=np.empty(0, dtype=[("LST", "f4")])) + return SimpleNamespace(store=store, schema={}, data_arr=np.empty(0, dtype=[("LST", "f4")])) def normalizer(spec, schema): normalizers.append(spec) @@ -97,7 +97,13 @@ def computed_stats(path): column[0, 3] = float("nan") monkeypatch.setattr(dataset.eodata, "get_data_reader", reader) monkeypatch.setattr(dataset.ModalityNormalizer, "from_spec", staticmethod(normalizer)) - monkeypatch.setattr(dataset, "_preload_columns", lambda *args, **kwargs: {"data": column}) + monkeypatch.setattr( + dataset, + "_preload_columns", + lambda reader, *args, **kwargs: { + "data": column + (100 if Path(reader.store).parent.name == "changed-data" else 0) + }, + ) monkeypatch.setattr(dataset, "load_computed_stats", computed_stats) split = dict( spatial="tiny", @@ -105,6 +111,8 @@ def computed_stats(path): stats_spatial="mtg_new", seviri=dict(store="tiny", load={"data": ["LST"]}, lookback=2, horizon=2, prognostics=["LST"]), ) + if explicit_paths: + split["seviri"]["store_root"] = "modality-data" for _ in range(2): module = LitDataModule( experiment_paths=paths if explicit_paths else None, @@ -118,8 +126,19 @@ def computed_stats(path): for data in (module.train_dataset, module.val_dataset, module.test_dataset): np.testing.assert_array_equal(data.indices[:, 1], dataset.STORE_EPOCH_OFFSET + np.array([2, 3])) torch.testing.assert_close(data[0]["seviri", "LST"].flatten(), column[0, :5, 0], equal_nan=True) - assert stores == [str(paths.eozarr_store_path("tiny"))] + expected_store = Path("modality-data/tiny.eozarr") if explicit_paths else paths.eozarr_store_path("tiny") + assert stores == [str(expected_store)] assert normalizers == [f"zscore:{stats}"] assert computed == [stats] * 6 assert len(list(paths.preload_cache_root.glob("preload_tiny_*.pt"))) == 1 assert dataset.PATHS is original_paths + if explicit_paths: + split["seviri"]["store_root"] = "changed-data" + changed = LitDataModule(experiment_paths=paths, **module.kwargs) + changed.setup() + torch.testing.assert_close( + changed.train_dataset[0]["seviri", "LST"].flatten(), column[0, :5, 0] + 100, equal_nan=True + ) + assert stores[-1] == str(Path("changed-data/tiny.eozarr")) + assert len(stores) == 2 + assert len(list(paths.preload_cache_root.glob("preload_tiny_*.pt"))) == 2 From f3ea9367be66246569def4315b62243ef1288cc8 Mon Sep 17 00:00:00 2001 From: ramin_yz Date: Fri, 13 Nov 2026 21:35:58 +0100 Subject: [PATCH 2/2] docs: explain per-modality store roots Document the optional source setting, its shared-root fallback and Hydra syntax. Explain when explicit modality roots separate preload caches. --- docs/CONFIGURATION.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3ab6c6d..b32e4d4 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -72,7 +72,8 @@ relative to the repository working directory. The overrides are saved in the run uv run python deep_lst/train.py paths.eozarr_root=/data/eozarr paths.preload_cache_root=/scratch/deep_lst-cache ``` -Cache keys are unchanged; use a separate cache root when the source data content differs. +Changing only `paths.eozarr_root` does not change cache keys; use a separate cache root when +the source data content differs. These Hydra overrides apply to training. Standalone inference, statistics and fine-tuning use the shared `ExperimentPaths` defaults, including the cache default described above. @@ -167,11 +168,24 @@ that training, inference, the baselines and the statistics script share. | Key under `sources.` | Meaning | |---|---| | `store` | name of the `.eozarr` store | +| `store_root` | optional; root directory for this modality's store. If omitted or `null`, use the shared `ExperimentPaths.eozarr_root` (`paths.eozarr_root` in training). | | `load` | variables read from the store, by data type (below). This is what reaches memory, not what the model sees; the model's variables are chosen in the `data` group and have to be listed here. | | `transform` | optional; a value transform applied before the normalisation. `signed_log1p` is sign(x) * log(1 + \|x\|), for variables that span orders of magnitude. The statistics have to be computed under the same transform; training checks this before it reads data. | | `increment_vars` | variables for which the statistics script also computes the statistics of the 1-hour increment. Read by `scripts/assets/compute_statistics.py` only. Listing a variable here does not make it a prognostic of the model. | | `quality_mask` | optional and not set: `{field, se_max, target}` sets `target` to missing where the metadata variable `field` exceeds `se_max` | +For example, keep ERA5 in the shared root while reading SEVIRI from another directory: + +``` +uv run python deep_lst/train.py +sources.seviri.store_root=/data/deep_lst +uv run scripts/assets/compute_statistics.py modality=seviri +sources.seviri.store_root=/data/deep_lst +``` + +The `+` adds this optional field in Hydra; it can also be added to the modality's source YAML. +The selected root contains `.eozarr` or `.eozarr.zip`, with the existing directory/ZIP +selection unchanged. Relative roots stay relative to the working directory. The source setting +is saved with the run config and follows the modality through the existing split views. + Data types under `load`: | Data type | Holds | @@ -197,6 +211,8 @@ only in the data options that name it. Changing `store`, `load` or `transform` changes the preload cache key, so the cache is rebuilt (`docs/RUNBOOK.md`, section 4.6). Changing the `data` option does not. +An explicit `store_root` is included in that key using its resolved path. Different roots use +separate caches; leaving `store_root` unset keeps the existing cache keys. ### 3.2 `data`