diff --git a/deep_lst/baselines/base.py b/deep_lst/baselines/base.py index fa6f863..9ac0209 100644 --- a/deep_lst/baselines/base.py +++ b/deep_lst/baselines/base.py @@ -86,7 +86,10 @@ def _preload_series(self, shared: dict, modality: str, var_name: str): def _metadata_attrs(self, modality: str, store_name: str, var_name: str) -> dict: """Baseline attrs sourced from the store's own per-variable metadata scheme.""" - reader = get_data_reader(modality, store=str(self.paths.eozarr_store_path(store_name)), average_hourly=False) + store_root = self.config["test_dataset"][modality].get("store_root") + reader = get_data_reader( + modality, store=str(self.paths.eozarr_store_path(store_name, store_root)), average_hourly=False + ) variable_meta = reader.data_arr.attrs.get("metadata", {}).get(var_name, {}) return dict( variable=var_name, diff --git a/deep_lst/dataset/dataset.py b/deep_lst/dataset/dataset.py index bc96dce..1a38beb 100644 --- a/deep_lst/dataset/dataset.py +++ b/deep_lst/dataset/dataset.py @@ -3,6 +3,7 @@ import json import os from concurrent.futures import ThreadPoolExecutor +from pathlib import Path import eodata import numpy as np @@ -242,7 +243,7 @@ def __getitem__(self, index): return leaves @classmethod - def from_config(cls, modality, store, spatial, anchor, temporal, workers=1): + def from_config(cls, modality, store, spatial, anchor, temporal, workers=1, store_root=None): """Stream the MTG pixel selection (spatial) over the statistics timestep subset (temporal).""" anchor_ids = _load_anchor_ids(spatial) rows = np.unique(map_to_store_rows(anchor_grid(anchor), anchor_ids, store_grid(modality))) @@ -260,7 +261,9 @@ def from_config(cls, modality, store, spatial, anchor, temporal, workers=1): } accepted = inspect.signature(get_data_reader_class(modality)).parameters reader_kwargs = {key: value for key, value in reader_kwargs.items() if key in accepted} - reader = eodata.get_data_reader(modality, store=str(PATHS.eozarr_store_path(store)), **reader_kwargs) + reader = eodata.get_data_reader( + modality, store=str(PATHS.eozarr_store_path(store, store_root)), **reader_kwargs + ) return cls(reader, rows, stat_days) @@ -406,6 +409,9 @@ def preload_shared(cls, modality_ids, paths=None, **kwargs): }, } for modality in modality_ids: + store_root = kwargs[modality].get("store_root") + if store_root is not None: + fingerprint_spec["modalities"][modality]["store_root"] = str(Path(store_root).resolve()) transform = kwargs[modality].get("transform") if transform is not None: # only when set, so caches built before the key existed stay valid fingerprint_spec["modalities"][modality]["transform"] = transform @@ -433,7 +439,9 @@ def preload_shared(cls, modality_ids, paths=None, **kwargs): accepted = inspect.signature(get_data_reader_class(modality)).parameters reader_kwargs = {key: value for key, value in reader_kwargs.items() if key in accepted} reader = eodata.get_data_reader( - modality, store=str(paths.eozarr_store_path(kwargs[modality]["store"])), **reader_kwargs + modality, + store=str(paths.eozarr_store_path(kwargs[modality]["store"], kwargs[modality].get("store_root"))), + **reader_kwargs, ) normalizer = ModalityNormalizer.from_spec( diff --git a/deep_lst/utils/io.py b/deep_lst/utils/io.py index 216a887..f809499 100644 --- a/deep_lst/utils/io.py +++ b/deep_lst/utils/io.py @@ -50,14 +50,15 @@ def config_path(self, experiment: str) -> Path: def statistics_path(self, modality, name) -> Path: return self.assets_root / modality / "statistics" / f"statistics_{name}.npz" - def eozarr_store_path(self, store_name: str) -> Path: + def eozarr_store_path(self, store_name: str, store_root: str | Path | None = None) -> Path: """Resolve a full eodata .eozarr store from its config-provided name. Prefers the directory store when it is readable and falls back to the .zip sibling: - under eozarr_root the era5, era5_land, glofas and swe stores exist only as zips, while + under the selected root the era5, era5_land, glofas and swe stores exist only as zips, while seviri is a directory store.""" - directory = self.eozarr_root / f"{store_name}.eozarr" - archive = self.eozarr_root / f"{store_name}.eozarr.zip" + root = self.eozarr_root if store_root is None else Path(store_root) + directory = root / f"{store_name}.eozarr" + archive = root / f"{store_name}.eozarr.zip" directory_is_readable = (directory / "data" / "zarr.json").exists() if not directory_is_readable and archive.exists(): return archive diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3ab6c6d..b32e4d4 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -72,7 +72,8 @@ relative to the repository working directory. The overrides are saved in the run uv run python deep_lst/train.py paths.eozarr_root=/data/eozarr paths.preload_cache_root=/scratch/deep_lst-cache ``` -Cache keys are unchanged; use a separate cache root when the source data content differs. +Changing only `paths.eozarr_root` does not change cache keys; use a separate cache root when +the source data content differs. These Hydra overrides apply to training. Standalone inference, statistics and fine-tuning use the shared `ExperimentPaths` defaults, including the cache default described above. @@ -167,11 +168,24 @@ that training, inference, the baselines and the statistics script share. | Key under `sources.` | Meaning | |---|---| | `store` | name of the `.eozarr` store | +| `store_root` | optional; root directory for this modality's store. If omitted or `null`, use the shared `ExperimentPaths.eozarr_root` (`paths.eozarr_root` in training). | | `load` | variables read from the store, by data type (below). This is what reaches memory, not what the model sees; the model's variables are chosen in the `data` group and have to be listed here. | | `transform` | optional; a value transform applied before the normalisation. `signed_log1p` is sign(x) * log(1 + \|x\|), for variables that span orders of magnitude. The statistics have to be computed under the same transform; training checks this before it reads data. | | `increment_vars` | variables for which the statistics script also computes the statistics of the 1-hour increment. Read by `scripts/assets/compute_statistics.py` only. Listing a variable here does not make it a prognostic of the model. | | `quality_mask` | optional and not set: `{field, se_max, target}` sets `target` to missing where the metadata variable `field` exceeds `se_max` | +For example, keep ERA5 in the shared root while reading SEVIRI from another directory: + +``` +uv run python deep_lst/train.py +sources.seviri.store_root=/data/deep_lst +uv run scripts/assets/compute_statistics.py modality=seviri +sources.seviri.store_root=/data/deep_lst +``` + +The `+` adds this optional field in Hydra; it can also be added to the modality's source YAML. +The selected root contains `.eozarr` or `.eozarr.zip`, with the existing directory/ZIP +selection unchanged. Relative roots stay relative to the working directory. The source setting +is saved with the run config and follows the modality through the existing split views. + Data types under `load`: | Data type | Holds | @@ -197,6 +211,8 @@ only in the data options that name it. Changing `store`, `load` or `transform` changes the preload cache key, so the cache is rebuilt (`docs/RUNBOOK.md`, section 4.6). Changing the `data` option does not. +An explicit `store_root` is included in that key using its resolved path. Different roots use +separate caches; leaving `store_root` unset keeps the existing cache keys. ### 3.2 `data` diff --git a/scripts/assets/compute_statistics.py b/scripts/assets/compute_statistics.py index 3637478..65c2c97 100644 --- a/scripts/assets/compute_statistics.py +++ b/scripts/assets/compute_statistics.py @@ -101,6 +101,7 @@ def main(cfg: DictConfig) -> None: dataset = UnimodalDataset.from_config( modality=modality, store=source["store"], + store_root=source.get("store_root"), spatial=config["spatial"], anchor=config["anchor"], temporal=config["temporal"], diff --git a/tests/test_config_golden.py b/tests/test_config_golden.py index e27d6be..d8e880c 100644 --- a/tests/test_config_golden.py +++ b/tests/test_config_golden.py @@ -213,6 +213,17 @@ def test_training_path_overrides_leave_scientific_config_unchanged(): assert overridden == default +def test_modality_store_root_reaches_each_split_without_changing_other_settings(): + default = _compose(NEW_OVERRIDES["default"]) + overridden = _compose([*NEW_OVERRIDES["default"], "+sources.seviri.store_root=local-seviri"]) + for view in config_utils.split_views(overridden).values(): + assert view["seviri"]["store_root"] == "local-seviri" + assert "store_root" not in view["era5"] + assert "store_root" not in view["era5_land"] + assert overridden["sources"]["seviri"].pop("store_root") == "local-seviri" + assert overridden == default + + @pytest.mark.parametrize("case", CASES) def test_migrated_fixture_equals_the_new_composition(case): migrated = config_utils.migrate_config(_fixture(case)) diff --git a/tests/test_io.py b/tests/test_io.py index 560776b..538591c 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -4,6 +4,8 @@ hermetic — nothing depends on the real on-disk layout. """ +from pathlib import Path + import pytest from deep_lst.utils.io import ExperimentPaths @@ -49,6 +51,26 @@ def test_explicit_cache_root_overrides_environment(monkeypatch, tmp_path): ) +@pytest.mark.parametrize("root_kind", ["default", "string", "path", "relative"]) +def test_store_root_override_keeps_directory_and_zip_selection(tmp_path, monkeypatch, root_kind): + monkeypatch.chdir(tmp_path) + paths = ExperimentPaths(eozarr_root=tmp_path / "shared") + root = paths.eozarr_root if root_kind == "default" else tmp_path / "modality" + if root_kind == "relative": + root = Path("modality") + kwargs = {} if root_kind == "default" else {"store_root": str(root) if root_kind == "string" else root} + directory = root / "tiny.eozarr" + archive = root / "tiny.eozarr.zip" + root.mkdir(parents=True) + assert paths.eozarr_store_path("tiny", **kwargs) == directory + archive.touch() + assert paths.eozarr_store_path("tiny", **kwargs) == archive + (directory / "data").mkdir(parents=True) + (directory / "data" / "zarr.json").write_text("{}") + assert paths.eozarr_store_path("tiny", **kwargs) == directory + assert paths.eozarr_root == tmp_path / "shared" + + # ── directory resolution ────────────────────────────────────────────────────── diff --git a/tests/test_reader_store_roots.py b/tests/test_reader_store_roots.py new file mode 100644 index 0000000..5e88f02 --- /dev/null +++ b/tests/test_reader_store_roots.py @@ -0,0 +1,89 @@ +"""Per-modality store roots for statistics and baseline metadata readers.""" + +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +from omegaconf import OmegaConf + +pytest.importorskip("eodata") +from deep_lst.baselines import base # noqa: E402 +from deep_lst.baselines.era5land import Era5LandExtractor # noqa: E402 +from deep_lst.dataset import dataset # noqa: E402 +from deep_lst.utils.io import ExperimentPaths # noqa: E402 +from scripts.assets import compute_statistics # noqa: E402 + + +@pytest.mark.parametrize("store_root", [None, "custom-store"]) +def test_statistics_dataset_uses_the_selected_store_root(tmp_path, monkeypatch, store_root): + monkeypatch.chdir(tmp_path) + paths = ExperimentPaths(assets_root=tmp_path / "assets", eozarr_root=tmp_path / "default-store") + for path, values in ( + (paths.spatial_indices_path("tiny"), {"src_indices": [0]}), + (paths.temporal_indices_path("tiny"), {"time_indices": [0, 24]}), + ): + path.parent.mkdir(parents=True, exist_ok=True) + np.savez(path, **values) + monkeypatch.setattr(dataset, "PATHS", paths) + monkeypatch.setattr(dataset, "anchor_grid", lambda _: None) + monkeypatch.setattr(dataset, "store_grid", lambda _: None) + monkeypatch.setattr(dataset, "map_to_store_rows", lambda *args: np.array([4])) + monkeypatch.setattr(dataset, "get_data_reader_class", lambda _: lambda store, average_hourly: None) + stores = [] + reader = SimpleNamespace() + + def open_reader(modality, store, **kwargs): + stores.append(store) + return reader + + monkeypatch.setattr(dataset.eodata, "get_data_reader", open_reader) + kwargs = {} if store_root is None else {"store_root": store_root} + result = dataset.UnimodalDataset.from_config("seviri", "tiny", "tiny", "mtg", "tiny", **kwargs) + expected_root = paths.eozarr_root if store_root is None else Path(store_root) + assert stores == [str(expected_root / "tiny.eozarr")] + assert result.reader is reader + np.testing.assert_array_equal(result.rows, [4]) + assert result.stat_days == [dataset.PRELOAD_START_DAY, dataset.PRELOAD_START_DAY + 1] + + +@pytest.mark.parametrize("store_root", [None, "custom-store"]) +def test_statistics_command_forwards_the_source_store_root(monkeypatch, store_root): + source = {"store": "tiny"} + if store_root is not None: + source["store_root"] = store_root + cfg = OmegaConf.create( + dict(modality="seviri", sources={"seviri": source}, spatial="tiny", anchor="mtg", temporal="tiny") + ) + + def stop_at_dataset(**kwargs): + assert kwargs["store"] == "tiny" + assert kwargs.get("store_root") == store_root + raise RuntimeError("statistics dataset reached") + + monkeypatch.setattr(compute_statistics.UnimodalDataset, "from_config", staticmethod(stop_at_dataset)) + with pytest.raises(RuntimeError, match="statistics dataset reached"): + compute_statistics.main.__wrapped__(cfg) + + +@pytest.mark.parametrize("store_root", [None, "custom-store"]) +def test_baseline_metadata_uses_the_modality_store_root(tmp_path, monkeypatch, store_root): + monkeypatch.chdir(tmp_path) + extractor = Era5LandExtractor("skt", 2024) + extractor.paths = ExperimentPaths(eozarr_root=tmp_path / "default-store") + source = {"store": "tiny"} + if store_root is not None: + source["store_root"] = store_root + extractor.config = {"test_dataset": {"era5_land": source}} + stores = [] + + def open_reader(modality, store, **kwargs): + assert modality == "era5_land" + stores.append(store) + return SimpleNamespace(data_arr=SimpleNamespace(attrs={"metadata": {"skt": {"units": "K"}}})) + + monkeypatch.setattr(base, "get_data_reader", open_reader) + attrs = extractor._metadata_attrs("era5_land", "tiny", "skt") + expected_root = extractor.paths.eozarr_root if store_root is None else Path(store_root) + assert stores == [str(expected_root / "tiny.eozarr")] + assert attrs == dict(variable="skt", units="K", long_name="skt", description="", test_year="2024") diff --git a/tests/test_training_paths.py b/tests/test_training_paths.py index 9e2e052..f96c6ce 100644 --- a/tests/test_training_paths.py +++ b/tests/test_training_paths.py @@ -84,7 +84,7 @@ def test_configured_reader_and_cache_preserve_samples_on_cold_and_warm_load(tmp_ def reader(modality, store, **kwargs): stores.append(store) - return SimpleNamespace(schema={}, data_arr=np.empty(0, dtype=[("LST", "f4")])) + return SimpleNamespace(store=store, schema={}, data_arr=np.empty(0, dtype=[("LST", "f4")])) def normalizer(spec, schema): normalizers.append(spec) @@ -97,7 +97,13 @@ def computed_stats(path): column[0, 3] = float("nan") monkeypatch.setattr(dataset.eodata, "get_data_reader", reader) monkeypatch.setattr(dataset.ModalityNormalizer, "from_spec", staticmethod(normalizer)) - monkeypatch.setattr(dataset, "_preload_columns", lambda *args, **kwargs: {"data": column}) + monkeypatch.setattr( + dataset, + "_preload_columns", + lambda reader, *args, **kwargs: { + "data": column + (100 if Path(reader.store).parent.name == "changed-data" else 0) + }, + ) monkeypatch.setattr(dataset, "load_computed_stats", computed_stats) split = dict( spatial="tiny", @@ -105,6 +111,8 @@ def computed_stats(path): stats_spatial="mtg_new", seviri=dict(store="tiny", load={"data": ["LST"]}, lookback=2, horizon=2, prognostics=["LST"]), ) + if explicit_paths: + split["seviri"]["store_root"] = "modality-data" for _ in range(2): module = LitDataModule( experiment_paths=paths if explicit_paths else None, @@ -118,8 +126,19 @@ def computed_stats(path): for data in (module.train_dataset, module.val_dataset, module.test_dataset): np.testing.assert_array_equal(data.indices[:, 1], dataset.STORE_EPOCH_OFFSET + np.array([2, 3])) torch.testing.assert_close(data[0]["seviri", "LST"].flatten(), column[0, :5, 0], equal_nan=True) - assert stores == [str(paths.eozarr_store_path("tiny"))] + expected_store = Path("modality-data/tiny.eozarr") if explicit_paths else paths.eozarr_store_path("tiny") + assert stores == [str(expected_store)] assert normalizers == [f"zscore:{stats}"] assert computed == [stats] * 6 assert len(list(paths.preload_cache_root.glob("preload_tiny_*.pt"))) == 1 assert dataset.PATHS is original_paths + if explicit_paths: + split["seviri"]["store_root"] = "changed-data" + changed = LitDataModule(experiment_paths=paths, **module.kwargs) + changed.setup() + torch.testing.assert_close( + changed.train_dataset[0]["seviri", "LST"].flatten(), column[0, :5, 0] + 100, equal_nan=True + ) + assert stores[-1] == str(Path("changed-data/tiny.eozarr")) + assert len(stores) == 2 + assert len(list(paths.preload_cache_root.glob("preload_tiny_*.pt"))) == 2