Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion deep_lst/baselines/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,10 @@ def _preload_series(self, shared: dict, modality: str, var_name: str):

def _metadata_attrs(self, modality: str, store_name: str, var_name: str) -> dict:
"""Baseline attrs sourced from the store's own per-variable metadata scheme."""
reader = get_data_reader(modality, store=str(self.paths.eozarr_store_path(store_name)), average_hourly=False)
store_root = self.config["test_dataset"][modality].get("store_root")
reader = get_data_reader(
modality, store=str(self.paths.eozarr_store_path(store_name, store_root)), average_hourly=False
)
variable_meta = reader.data_arr.attrs.get("metadata", {}).get(var_name, {})
return dict(
variable=var_name,
Expand Down
14 changes: 11 additions & 3 deletions deep_lst/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import json
import os
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import eodata
import numpy as np
Expand Down Expand Up @@ -242,7 +243,7 @@ def __getitem__(self, index):
return leaves

@classmethod
def from_config(cls, modality, store, spatial, anchor, temporal, workers=1):
def from_config(cls, modality, store, spatial, anchor, temporal, workers=1, store_root=None):
"""Stream the MTG pixel selection (spatial) over the statistics timestep subset (temporal)."""
anchor_ids = _load_anchor_ids(spatial)
rows = np.unique(map_to_store_rows(anchor_grid(anchor), anchor_ids, store_grid(modality)))
Expand All @@ -260,7 +261,9 @@ def from_config(cls, modality, store, spatial, anchor, temporal, workers=1):
}
accepted = inspect.signature(get_data_reader_class(modality)).parameters
reader_kwargs = {key: value for key, value in reader_kwargs.items() if key in accepted}
reader = eodata.get_data_reader(modality, store=str(PATHS.eozarr_store_path(store)), **reader_kwargs)
reader = eodata.get_data_reader(
modality, store=str(PATHS.eozarr_store_path(store, store_root)), **reader_kwargs
)
return cls(reader, rows, stat_days)


Expand Down Expand Up @@ -406,6 +409,9 @@ def preload_shared(cls, modality_ids, paths=None, **kwargs):
},
}
for modality in modality_ids:
store_root = kwargs[modality].get("store_root")
if store_root is not None:
fingerprint_spec["modalities"][modality]["store_root"] = str(Path(store_root).resolve())
transform = kwargs[modality].get("transform")
if transform is not None: # only when set, so caches built before the key existed stay valid
fingerprint_spec["modalities"][modality]["transform"] = transform
Expand Down Expand Up @@ -433,7 +439,9 @@ def preload_shared(cls, modality_ids, paths=None, **kwargs):
accepted = inspect.signature(get_data_reader_class(modality)).parameters
reader_kwargs = {key: value for key, value in reader_kwargs.items() if key in accepted}
reader = eodata.get_data_reader(
modality, store=str(paths.eozarr_store_path(kwargs[modality]["store"])), **reader_kwargs
modality,
store=str(paths.eozarr_store_path(kwargs[modality]["store"], kwargs[modality].get("store_root"))),
**reader_kwargs,
)

normalizer = ModalityNormalizer.from_spec(
Expand Down
9 changes: 5 additions & 4 deletions deep_lst/utils/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,15 @@ def config_path(self, experiment: str) -> Path:
def statistics_path(self, modality, name) -> Path:
return self.assets_root / modality / "statistics" / f"statistics_{name}.npz"

def eozarr_store_path(self, store_name: str) -> Path:
def eozarr_store_path(self, store_name: str, store_root: str | Path | None = None) -> Path:
"""Resolve a full eodata .eozarr store from its config-provided name.

Prefers the directory store when it is readable and falls back to the .zip sibling:
under eozarr_root the era5, era5_land, glofas and swe stores exist only as zips, while
under the selected root the era5, era5_land, glofas and swe stores exist only as zips, while
seviri is a directory store."""
directory = self.eozarr_root / f"{store_name}.eozarr"
archive = self.eozarr_root / f"{store_name}.eozarr.zip"
root = self.eozarr_root if store_root is None else Path(store_root)
directory = root / f"{store_name}.eozarr"
archive = root / f"{store_name}.eozarr.zip"
directory_is_readable = (directory / "data" / "zarr.json").exists()
if not directory_is_readable and archive.exists():
return archive
Expand Down
18 changes: 17 additions & 1 deletion docs/CONFIGURATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ relative to the repository working directory. The overrides are saved in the run
uv run python deep_lst/train.py paths.eozarr_root=/data/eozarr paths.preload_cache_root=/scratch/deep_lst-cache
```

Cache keys are unchanged; use a separate cache root when the source data content differs.
Changing only `paths.eozarr_root` does not change cache keys; use a separate cache root when
the source data content differs.

These Hydra overrides apply to training. Standalone inference, statistics and fine-tuning use
the shared `ExperimentPaths` defaults, including the cache default described above.
Expand Down Expand Up @@ -167,11 +168,24 @@ that training, inference, the baselines and the statistics script share.
| Key under `sources.<modality>` | Meaning |
|---|---|
| `store` | name of the `.eozarr` store |
| `store_root` | optional; root directory for this modality's store. If omitted or `null`, use the shared `ExperimentPaths.eozarr_root` (`paths.eozarr_root` in training). |
| `load` | variables read from the store, by data type (below). This is what reaches memory, not what the model sees; the model's variables are chosen in the `data` group and have to be listed here. |
| `transform` | optional; a value transform applied before the normalisation. `signed_log1p` is sign(x) * log(1 + \|x\|), for variables that span orders of magnitude. The statistics have to be computed under the same transform; training checks this before it reads data. |
| `increment_vars` | variables for which the statistics script also computes the statistics of the 1-hour increment. Read by `scripts/assets/compute_statistics.py` only. Listing a variable here does not make it a prognostic of the model. |
| `quality_mask` | optional and not set: `{field, se_max, target}` sets `target` to missing where the metadata variable `field` exceeds `se_max` |

For example, keep ERA5 in the shared root while reading SEVIRI from another directory:

```
uv run python deep_lst/train.py +sources.seviri.store_root=/data/deep_lst
uv run scripts/assets/compute_statistics.py modality=seviri +sources.seviri.store_root=/data/deep_lst
```

The `+` adds this optional field in Hydra; it can also be added to the modality's source YAML.
The selected root contains `<store>.eozarr` or `<store>.eozarr.zip`, with the existing directory/ZIP
selection unchanged. Relative roots stay relative to the working directory. The source setting
is saved with the run config and follows the modality through the existing split views.

Data types under `load`:

| Data type | Holds |
Expand All @@ -197,6 +211,8 @@ only in the data options that name it.

Changing `store`, `load` or `transform` changes the preload cache key, so the cache is rebuilt
(`docs/RUNBOOK.md`, section 4.6). Changing the `data` option does not.
An explicit `store_root` is included in that key using its resolved path. Different roots use
separate caches; leaving `store_root` unset keeps the existing cache keys.

### 3.2 `data`

Expand Down
1 change: 1 addition & 0 deletions scripts/assets/compute_statistics.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ def main(cfg: DictConfig) -> None:
dataset = UnimodalDataset.from_config(
modality=modality,
store=source["store"],
store_root=source.get("store_root"),
spatial=config["spatial"],
anchor=config["anchor"],
temporal=config["temporal"],
Expand Down
11 changes: 11 additions & 0 deletions tests/test_config_golden.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,17 @@ def test_training_path_overrides_leave_scientific_config_unchanged():
assert overridden == default


def test_modality_store_root_reaches_each_split_without_changing_other_settings():
default = _compose(NEW_OVERRIDES["default"])
overridden = _compose([*NEW_OVERRIDES["default"], "+sources.seviri.store_root=local-seviri"])
for view in config_utils.split_views(overridden).values():
assert view["seviri"]["store_root"] == "local-seviri"
assert "store_root" not in view["era5"]
assert "store_root" not in view["era5_land"]
assert overridden["sources"]["seviri"].pop("store_root") == "local-seviri"
assert overridden == default


@pytest.mark.parametrize("case", CASES)
def test_migrated_fixture_equals_the_new_composition(case):
migrated = config_utils.migrate_config(_fixture(case))
Expand Down
22 changes: 22 additions & 0 deletions tests/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
hermetic — nothing depends on the real on-disk layout.
"""

from pathlib import Path

import pytest

from deep_lst.utils.io import ExperimentPaths
Expand Down Expand Up @@ -49,6 +51,26 @@ def test_explicit_cache_root_overrides_environment(monkeypatch, tmp_path):
)


@pytest.mark.parametrize("root_kind", ["default", "string", "path", "relative"])
def test_store_root_override_keeps_directory_and_zip_selection(tmp_path, monkeypatch, root_kind):
monkeypatch.chdir(tmp_path)
paths = ExperimentPaths(eozarr_root=tmp_path / "shared")
root = paths.eozarr_root if root_kind == "default" else tmp_path / "modality"
if root_kind == "relative":
root = Path("modality")
kwargs = {} if root_kind == "default" else {"store_root": str(root) if root_kind == "string" else root}
directory = root / "tiny.eozarr"
archive = root / "tiny.eozarr.zip"
root.mkdir(parents=True)
assert paths.eozarr_store_path("tiny", **kwargs) == directory
archive.touch()
assert paths.eozarr_store_path("tiny", **kwargs) == archive
(directory / "data").mkdir(parents=True)
(directory / "data" / "zarr.json").write_text("{}")
assert paths.eozarr_store_path("tiny", **kwargs) == directory
assert paths.eozarr_root == tmp_path / "shared"


# ── directory resolution ──────────────────────────────────────────────────────


Expand Down
89 changes: 89 additions & 0 deletions tests/test_reader_store_roots.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
"""Per-modality store roots for statistics and baseline metadata readers."""

from pathlib import Path
from types import SimpleNamespace

import numpy as np
import pytest
from omegaconf import OmegaConf

pytest.importorskip("eodata")
from deep_lst.baselines import base # noqa: E402
from deep_lst.baselines.era5land import Era5LandExtractor # noqa: E402
from deep_lst.dataset import dataset # noqa: E402
from deep_lst.utils.io import ExperimentPaths # noqa: E402
from scripts.assets import compute_statistics # noqa: E402


@pytest.mark.parametrize("store_root", [None, "custom-store"])
def test_statistics_dataset_uses_the_selected_store_root(tmp_path, monkeypatch, store_root):
monkeypatch.chdir(tmp_path)
paths = ExperimentPaths(assets_root=tmp_path / "assets", eozarr_root=tmp_path / "default-store")
for path, values in (
(paths.spatial_indices_path("tiny"), {"src_indices": [0]}),
(paths.temporal_indices_path("tiny"), {"time_indices": [0, 24]}),
):
path.parent.mkdir(parents=True, exist_ok=True)
np.savez(path, **values)
monkeypatch.setattr(dataset, "PATHS", paths)
monkeypatch.setattr(dataset, "anchor_grid", lambda _: None)
monkeypatch.setattr(dataset, "store_grid", lambda _: None)
monkeypatch.setattr(dataset, "map_to_store_rows", lambda *args: np.array([4]))
monkeypatch.setattr(dataset, "get_data_reader_class", lambda _: lambda store, average_hourly: None)
stores = []
reader = SimpleNamespace()

def open_reader(modality, store, **kwargs):
stores.append(store)
return reader

monkeypatch.setattr(dataset.eodata, "get_data_reader", open_reader)
kwargs = {} if store_root is None else {"store_root": store_root}
result = dataset.UnimodalDataset.from_config("seviri", "tiny", "tiny", "mtg", "tiny", **kwargs)
expected_root = paths.eozarr_root if store_root is None else Path(store_root)
assert stores == [str(expected_root / "tiny.eozarr")]
assert result.reader is reader
np.testing.assert_array_equal(result.rows, [4])
assert result.stat_days == [dataset.PRELOAD_START_DAY, dataset.PRELOAD_START_DAY + 1]


@pytest.mark.parametrize("store_root", [None, "custom-store"])
def test_statistics_command_forwards_the_source_store_root(monkeypatch, store_root):
source = {"store": "tiny"}
if store_root is not None:
source["store_root"] = store_root
cfg = OmegaConf.create(
dict(modality="seviri", sources={"seviri": source}, spatial="tiny", anchor="mtg", temporal="tiny")
)

def stop_at_dataset(**kwargs):
assert kwargs["store"] == "tiny"
assert kwargs.get("store_root") == store_root
raise RuntimeError("statistics dataset reached")

monkeypatch.setattr(compute_statistics.UnimodalDataset, "from_config", staticmethod(stop_at_dataset))
with pytest.raises(RuntimeError, match="statistics dataset reached"):
compute_statistics.main.__wrapped__(cfg)


@pytest.mark.parametrize("store_root", [None, "custom-store"])
def test_baseline_metadata_uses_the_modality_store_root(tmp_path, monkeypatch, store_root):
monkeypatch.chdir(tmp_path)
extractor = Era5LandExtractor("skt", 2024)
extractor.paths = ExperimentPaths(eozarr_root=tmp_path / "default-store")
source = {"store": "tiny"}
if store_root is not None:
source["store_root"] = store_root
extractor.config = {"test_dataset": {"era5_land": source}}
stores = []

def open_reader(modality, store, **kwargs):
assert modality == "era5_land"
stores.append(store)
return SimpleNamespace(data_arr=SimpleNamespace(attrs={"metadata": {"skt": {"units": "K"}}}))

monkeypatch.setattr(base, "get_data_reader", open_reader)
attrs = extractor._metadata_attrs("era5_land", "tiny", "skt")
expected_root = extractor.paths.eozarr_root if store_root is None else Path(store_root)
assert stores == [str(expected_root / "tiny.eozarr")]
assert attrs == dict(variable="skt", units="K", long_name="skt", description="", test_year="2024")
25 changes: 22 additions & 3 deletions tests/test_training_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def test_configured_reader_and_cache_preserve_samples_on_cold_and_warm_load(tmp_

def reader(modality, store, **kwargs):
stores.append(store)
return SimpleNamespace(schema={}, data_arr=np.empty(0, dtype=[("LST", "f4")]))
return SimpleNamespace(store=store, schema={}, data_arr=np.empty(0, dtype=[("LST", "f4")]))

def normalizer(spec, schema):
normalizers.append(spec)
Expand All @@ -97,14 +97,22 @@ def computed_stats(path):
column[0, 3] = float("nan")
monkeypatch.setattr(dataset.eodata, "get_data_reader", reader)
monkeypatch.setattr(dataset.ModalityNormalizer, "from_spec", staticmethod(normalizer))
monkeypatch.setattr(dataset, "_preload_columns", lambda *args, **kwargs: {"data": column})
monkeypatch.setattr(
dataset,
"_preload_columns",
lambda reader, *args, **kwargs: {
"data": column + (100 if Path(reader.store).parent.name == "changed-data" else 0)
},
)
monkeypatch.setattr(dataset, "load_computed_stats", computed_stats)
split = dict(
spatial="tiny",
temporal="tiny",
stats_spatial="mtg_new",
seviri=dict(store="tiny", load={"data": ["LST"]}, lookback=2, horizon=2, prognostics=["LST"]),
)
if explicit_paths:
split["seviri"]["store_root"] = "modality-data"
for _ in range(2):
module = LitDataModule(
experiment_paths=paths if explicit_paths else None,
Expand All @@ -118,8 +126,19 @@ def computed_stats(path):
for data in (module.train_dataset, module.val_dataset, module.test_dataset):
np.testing.assert_array_equal(data.indices[:, 1], dataset.STORE_EPOCH_OFFSET + np.array([2, 3]))
torch.testing.assert_close(data[0]["seviri", "LST"].flatten(), column[0, :5, 0], equal_nan=True)
assert stores == [str(paths.eozarr_store_path("tiny"))]
expected_store = Path("modality-data/tiny.eozarr") if explicit_paths else paths.eozarr_store_path("tiny")
assert stores == [str(expected_store)]
assert normalizers == [f"zscore:{stats}"]
assert computed == [stats] * 6
assert len(list(paths.preload_cache_root.glob("preload_tiny_*.pt"))) == 1
assert dataset.PATHS is original_paths
if explicit_paths:
split["seviri"]["store_root"] = "changed-data"
changed = LitDataModule(experiment_paths=paths, **module.kwargs)
changed.setup()
torch.testing.assert_close(
changed.train_dataset[0]["seviri", "LST"].flatten(), column[0, :5, 0] + 100, equal_nan=True
)
assert stores[-1] == str(Path("changed-data/tiny.eozarr"))
assert len(stores) == 2
assert len(list(paths.preload_cache_root.glob("preload_tiny_*.pt"))) == 2
Loading