From 6e5cece3f47f3f27d5e05f8fb88ec9c62c9830ce Mon Sep 17 00:00:00 2001 From: ramin_yz Date: Tue, 10 Nov 2026 17:06:49 +0100 Subject: [PATCH 1/4] feat: configure training data and preload cache roots Expose optional data/cache root overrides in the existing Hydra training config and pass one ExperimentPaths instance through the DataModule and multimodal dataset. Keep null defaults, working-directory-relative paths, cache fingerprints and scientific settings unchanged. Add regression checks for independent overrides, saved run configuration, reader/statistics path wiring, and cold/warm cache sample equality. Reuse existing config and rollout-loss golden fixtures without changing them. Validation on the final working tree: 217 passed, 2 skipped; all six pre-commit hooks passed. All 137 passing pre-change baseline tests still pass. Real institutional-store training equivalence remains untested. --- conf/train.yaml | 4 ++ deep_lst/dataset/data_module.py | 9 ++- deep_lst/dataset/dataset.py | 25 ++++--- deep_lst/train.py | 5 +- tests/test_config_golden.py | 12 +++- tests/test_training_paths.py | 124 ++++++++++++++++++++++++++++++++ 6 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 tests/test_training_paths.py diff --git a/conf/train.yaml b/conf/train.yaml index f71aa37..a13662a 100644 --- a/conf/train.yaml +++ b/conf/train.yaml @@ -11,6 +11,10 @@ defaults: config_version: 2 +paths: + eozarr_root: null + preload_cache_root: null + # --- derived keys (resolved lazily by OmegaConf) --- _spatial: mtg_new _spatial_val: eval_mtg_new_val diff --git a/deep_lst/dataset/data_module.py b/deep_lst/dataset/data_module.py index a6072e2..e98543c 100644 --- a/deep_lst/dataset/data_module.py +++ b/deep_lst/dataset/data_module.py @@ -7,9 +7,10 @@ class LitDataModule(pl.LightningDataModule): - def __init__(self, **kwargs): + def __init__(self, experiment_paths=None, **kwargs): super().__init__() self.kwargs = kwargs + self.paths = experiment_paths self.collate_fn = Collator(kwargs["model"]["name"], get_packing_keys(kwargs["train_dataset"])) def setup(self, stage=None): @@ -27,9 +28,11 @@ def _build_split(self, split_kwargs): modality_ids = self.kwargs["modality_ids"] spatial = split_kwargs["spatial"] if spatial not in self._shared_by_spatial: - self._shared_by_spatial[spatial] = MultimodalDataset.preload_shared(modality_ids, **split_kwargs) + self._shared_by_spatial[spatial] = MultimodalDataset.preload_shared( + modality_ids, paths=self.paths, **split_kwargs + ) shared = self._shared_by_spatial[spatial] - return MultimodalDataset.from_modalities(modality_ids, shared, **split_kwargs) + return MultimodalDataset.from_modalities(modality_ids, shared, paths=self.paths, **split_kwargs) def train_dataloader(self): return DataLoader( diff --git a/deep_lst/dataset/dataset.py b/deep_lst/dataset/dataset.py index 650ed7c..bc96dce 100644 --- a/deep_lst/dataset/dataset.py +++ b/deep_lst/dataset/dataset.py @@ -75,10 +75,11 @@ def load_samples( return indices -def _load_anchor_ids(spatial_name): +def _load_anchor_ids(spatial_name, paths=None): """Anchor cell ids for a split: global MTG ids for training, seviri equiangular pixels for an eval holdout region.""" - return np.sort(np.load(PATHS.spatial_indices_path(spatial_name))["src_indices"].astype(np.int64)) + paths = paths or PATHS + return np.sort(np.load(paths.spatial_indices_path(spatial_name))["src_indices"].astype(np.int64)) def _flatten_hours(values, valid): @@ -373,7 +374,7 @@ def non_land_mask(self, classes=(17, 18)) -> np.ndarray: return np.isin(landcov[self.inverse[_ANCHOR_MODALITY]], classes) @classmethod - def preload_shared(cls, modality_ids, **kwargs): + def preload_shared(cls, modality_ids, paths=None, **kwargs): """Preload each modality's native columns once, shared across train/val/test splits. Every anchor cell (MTG for training, seviri pixels for eval) is nearest-neighbour mapped @@ -381,9 +382,10 @@ def preload_shared(cls, modality_ids, **kwargs): are read. Columns depend only on (anchors, store), not the temporal split, so this runs once — a full-axis cache per split would not fit in RAM. """ + paths = paths or PATHS spatial_name = kwargs["spatial"] stats_tag = stats_spatial(kwargs) - anchor_ids = _load_anchor_ids(spatial_name) + anchor_ids = _load_anchor_ids(spatial_name, paths) src_grid = anchor_grid(kwargs.get("anchor", "mtg")) num_workers = kwargs.get("preload_workers", 8) @@ -397,7 +399,7 @@ def preload_shared(cls, modality_ids, **kwargs): modality: { "store": kwargs[modality]["store"], "load": kwargs[modality]["load"], - "stats_mtime": os.path.getmtime(PATHS.stats_zarr_path(modality, stats_tag) / "zarr.json"), + "stats_mtime": os.path.getmtime(paths.stats_zarr_path(modality, stats_tag) / "zarr.json"), "axis": [PRELOAD_START_DAY, PRELOAD_END_DAY], } for modality in modality_ids @@ -408,7 +410,7 @@ def preload_shared(cls, modality_ids, **kwargs): if transform is not None: # only when set, so caches built before the key existed stay valid fingerprint_spec["modalities"][modality]["transform"] = transform fingerprint = hashlib.sha256(json.dumps(fingerprint_spec, sort_keys=True).encode()).hexdigest()[:16] - cache_path = PATHS.preload_cache_path(fingerprint, spatial_name) + cache_path = paths.preload_cache_path(fingerprint, spatial_name) if cache_path.exists(): print(f"Loading preload cache {cache_path}") return torch.load(cache_path, map_location="cpu", weights_only=False) @@ -431,11 +433,11 @@ def preload_shared(cls, modality_ids, **kwargs): accepted = inspect.signature(get_data_reader_class(modality)).parameters reader_kwargs = {key: value for key, value in reader_kwargs.items() if key in accepted} reader = eodata.get_data_reader( - modality, store=str(PATHS.eozarr_store_path(kwargs[modality]["store"])), **reader_kwargs + modality, store=str(paths.eozarr_store_path(kwargs[modality]["store"])), **reader_kwargs ) normalizer = ModalityNormalizer.from_spec( - f"zscore:{PATHS.stats_zarr_path(modality, stats_tag)}", reader.schema + f"zscore:{paths.stats_zarr_path(modality, stats_tag)}", reader.schema ) normalizer = with_transform(normalizer, kwargs[modality].get("transform")) # Channel indices of the config variables within each data type's full store variable order. @@ -462,17 +464,18 @@ def preload_shared(cls, modality_ids, **kwargs): return shared @classmethod - def from_modalities(cls, modality_ids, shared, **kwargs): + def from_modalities(cls, modality_ids, shared, paths=None, **kwargs): """Build one split's dataset from the shared preloaded columns + this split's anchors.""" + paths = paths or PATHS stats_tag = stats_spatial(kwargs) - computed_stats = {m: load_computed_stats(PATHS.stats_zarr_path(m, stats_tag)) for m in modality_ids} + computed_stats = {m: load_computed_stats(paths.stats_zarr_path(m, stats_tag)) for m in modality_ids} if kwargs["spatial"] != shared["spatial"]: raise ValueError(f"Shared cache built for spatial={shared['spatial']!r}, got {kwargs['spatial']!r}") temporal_split = kwargs["temporal"] if temporal_split is not None: - temporal_paths = PATHS.temporal_indices_path(temporal_split) + temporal_paths = paths.temporal_indices_path(temporal_split) else: temporal_paths = None diff --git a/deep_lst/train.py b/deep_lst/train.py index da9f64b..00b169f 100644 --- a/deep_lst/train.py +++ b/deep_lst/train.py @@ -45,11 +45,10 @@ def _git_hash() -> str: @hydra.main(config_path="../conf", config_name="train", version_base="1.3") def main(cfg: DictConfig) -> None: - paths = ExperimentPaths() - config = OmegaConf.to_container(cfg, resolve=True) choices = HydraConfig.get().runtime.choices validate_config(config) + paths = ExperimentPaths(**{key: Path(value) for key, value in config.get("paths", {}).items() if value is not None}) pl.seed_everything(config["seed"], workers=True) smoke = config.pop("smoke", False) @@ -156,7 +155,7 @@ def main(cfg: DictConfig) -> None: check_stats_transform(paths.stats_zarr_path(modality, stats_tag), transform) data_module = LitDataModule( - **config, train_dataset=train_view, val_dataset=views["val"], test_dataset=views["test"] + experiment_paths=paths, **config, train_dataset=train_view, val_dataset=views["val"], test_dataset=views["test"] ) data_module.setup() diff --git a/tests/test_config_golden.py b/tests/test_config_golden.py index c3f292c..e27d6be 100644 --- a/tests/test_config_golden.py +++ b/tests/test_config_golden.py @@ -111,7 +111,7 @@ "_sigma_prognostics", "_sigma_diagnostics", ] -NEW_TOP_LEVEL = ["config_version", "objective", "optim", "window", "sources", "features", "splits"] +NEW_TOP_LEVEL = ["config_version", "paths", "objective", "optim", "window", "sources", "features", "splits"] # the sections migrate_config has to rebuild exactly MIGRATED_SECTIONS = [ "config_version", @@ -203,6 +203,16 @@ def test_every_other_key_is_unchanged(case): assert sorted(new) == sorted(untouched + ["model"] + NEW_TOP_LEVEL) +def test_training_path_overrides_leave_scientific_config_unchanged(): + default = _compose(NEW_OVERRIDES["default"]) + overridden = _compose( + [*NEW_OVERRIDES["default"], "paths.eozarr_root=local-data", "paths.preload_cache_root=local-cache"] + ) + assert default.pop("paths") == {"eozarr_root": None, "preload_cache_root": None} + assert overridden.pop("paths") == {"eozarr_root": "local-data", "preload_cache_root": "local-cache"} + assert overridden == default + + @pytest.mark.parametrize("case", CASES) def test_migrated_fixture_equals_the_new_composition(case): migrated = config_utils.migrate_config(_fixture(case)) diff --git a/tests/test_training_paths.py b/tests/test_training_paths.py new file mode 100644 index 0000000..b31aa0e --- /dev/null +++ b/tests/test_training_paths.py @@ -0,0 +1,124 @@ +"""Training path wiring, with tiny inputs instead of the institutional stores.""" + +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from hydra import compose, initialize_config_dir +from omegaconf import OmegaConf + +pytest.importorskip("eodata") +from deep_lst import train # noqa: E402 +from deep_lst.dataset import dataset # noqa: E402 +from deep_lst.dataset.data_module import LitDataModule # noqa: E402 +from deep_lst.utils.io import ExperimentPaths # noqa: E402 + +CONF_DIR = Path(__file__).resolve().parents[1] / "conf" + + +@pytest.mark.parametrize( + "overrides", + [ + {}, + {"eozarr_root": "local-data"}, + {"preload_cache_root": "local-cache"}, + {"eozarr_root": "local-data", "preload_cache_root": "local-cache"}, + ], +) +def test_training_passes_paths_to_data_module_and_saves_overrides(tmp_path, monkeypatch, overrides): + with initialize_config_dir(config_dir=str(CONF_DIR), version_base="1.3"): + cfg = compose( + config_name="train", overrides=["wandb_logger=false", *[f"paths.{k}={v}" for k, v in overrides.items()]] + ) + monkeypatch.chdir(tmp_path) + choices = dict(model="mlp", objective="rollout", loss="smoothl1", horizon="h24", data="full") + monkeypatch.setattr(train.HydraConfig, "get", lambda: SimpleNamespace(runtime=SimpleNamespace(choices=choices))) + monkeypatch.setattr(train, "check_stats_transform", lambda *args: None) + monkeypatch.setattr(train.pl, "seed_everything", lambda *args, **kwargs: None) + monkeypatch.setattr(train.torch, "set_float32_matmul_precision", lambda *args: None) + received = [] + + class DataModuleReached(Exception): + pass + + def stop_at_data_module(**kwargs): + received.append(kwargs.get("experiment_paths")) + raise DataModuleReached + + monkeypatch.setattr(train, "LitDataModule", stop_at_data_module) + with pytest.raises(DataModuleReached): + train.main.__wrapped__(cfg) + assert received == [ExperimentPaths(**{k: Path(v) for k, v in overrides.items()})] + saved = OmegaConf.load(next(tmp_path.glob("experiments/*/version_*/config.yml"))) + assert saved.paths == cfg.paths + assert not (tmp_path / "local-data").exists() + + +@pytest.mark.parametrize("explicit_paths", [False, True]) +def test_configured_reader_and_cache_preserve_samples_on_cold_and_warm_load(tmp_path, monkeypatch, explicit_paths): + if explicit_paths: + monkeypatch.chdir(tmp_path) + root = Path(".") if explicit_paths else tmp_path + paths = ExperimentPaths(assets_root=root / "assets", eozarr_root=root / "data", preload_cache_root=root / "cache") + spatial = paths.spatial_indices_path("tiny") + spatial.parent.mkdir(parents=True) + np.savez(spatial, src_indices=[0]) + temporal = paths.temporal_indices_path("tiny") + temporal.parent.mkdir(parents=True) + np.savez(temporal, time_indices=[2, 3]) + stats = paths.stats_zarr_path("seviri", "mtg_new") + stats.mkdir(parents=True) + (stats / "zarr.json").write_text("{}") + if not explicit_paths: + monkeypatch.setattr(dataset, "PATHS", paths) + original_paths = dataset.PATHS + grid = SimpleNamespace(centers=np.array([[20.0, 10.0]])) + monkeypatch.setattr(dataset, "anchor_grid", lambda _: grid) + monkeypatch.setattr(dataset, "store_grid", lambda _: grid) + monkeypatch.setattr(dataset, "map_to_store_rows", lambda *args: np.array([0])) + monkeypatch.setattr(dataset, "get_data_reader_class", lambda _: lambda store, average_hourly: None) + stores, normalizers, computed = [], [], [] + + def reader(modality, store, **kwargs): + stores.append(store) + return SimpleNamespace(schema={}, data_arr=np.empty(0, dtype=[("LST", "f4")])) + + def normalizer(spec, schema): + normalizers.append(spec) + + def computed_stats(path): + computed.append(path) + return {"solar_zenith": (0.0, 1.0)} + + column = torch.arange(48, dtype=torch.float32).reshape(1, 48, 1) + column[0, 3] = float("nan") + monkeypatch.setattr(dataset.eodata, "get_data_reader", reader) + monkeypatch.setattr(dataset.ModalityNormalizer, "from_spec", staticmethod(normalizer)) + monkeypatch.setattr(dataset, "_preload_columns", lambda *args, **kwargs: {"data": column}) + monkeypatch.setattr(dataset, "load_computed_stats", computed_stats) + split = dict( + spatial="tiny", + temporal="tiny", + stats_spatial="mtg_new", + seviri=dict(store="tiny", load={"data": ["LST"]}, lookback=2, horizon=2, prognostics=["LST"]), + ) + for _ in range(2): + module = LitDataModule( + experiment_paths=paths if explicit_paths else None, + modality_ids=["seviri"], + model={"name": "mlp"}, + train_dataset=split, + val_dataset=split, + test_dataset=split, + ) + module.setup() + for data in (module.train_dataset, module.val_dataset, module.test_dataset): + np.testing.assert_array_equal(data.indices[:, 1], dataset.STORE_EPOCH_OFFSET + np.array([2, 3])) + torch.testing.assert_close(data[0]["seviri", "LST"].flatten(), column[0, :5, 0], equal_nan=True) + assert stores == [str(paths.eozarr_store_path("tiny"))] + assert normalizers == [f"zscore:{stats}"] + assert computed == [stats] * 6 + assert len(list(paths.preload_cache_root.glob("preload_tiny_*.pt"))) == 1 + assert dataset.PATHS is original_paths From 51ad26e22271bc3eb7c72636431d6e9f2cc4399f Mon Sep 17 00:00:00 2001 From: ramin_yz Date: Tue, 10 Nov 2026 17:07:00 +0100 Subject: [PATCH 2/4] docs: explain training path overrides and compatibility Document CLI/config examples, preserved defaults and relative-path semantics, plus the saved run configuration. Replace source-edit advice for training with Hydra overrides. State the training-only scope and unchanged cache-key behavior: different data contents should use separate cache roots. Other entry points keep their existing defaults. Validation: documentation was included in the passing final pre-commit run. No production or test content changed after the 217-pass suite. --- docs/CONFIGURATION.md | 16 ++++++++++++++++ docs/RUNBOOK.md | 6 ++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index cb2e515..b009cf9 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -56,6 +56,22 @@ be edited. ## 2. `conf/train.yaml` +### Training data and cache paths + +`paths.eozarr_root` and `paths.preload_cache_root` override the data-store and preload-cache +roots used by `deep_lst/train.py`, for all three training/validation/test splits. Both default +to `null`, which keeps the existing `ExperimentPaths` defaults. Relative paths remain relative +to the repository working directory. The overrides are saved in the run's `config.yml`. + +``` +uv run python deep_lst/train.py paths.eozarr_root=/data/eozarr paths.preload_cache_root=/scratch/deep_lst-cache +``` + +Cache keys are unchanged; use a separate cache root when the source data content differs. + +This first slice applies to training; standalone inference, statistics and fine-tuning entry +points still use their existing paths. Model, loss and data-processing settings are unchanged. + ### 2.1 Selections and splits | Key | Default | Meaning | diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index d56fc82..a29d56a 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -108,8 +108,10 @@ assets/baselines//.nc persistence / climatology / ERA5(-Land Two hard-coded paths need attention in a new clone: -1. `ExperimentPaths.preload_cache_root` in `deep_lst/utils/io.py` points at the maintainer's - tscratch directory, which you cannot write to. Point it at your own tscratch directory. +1. `ExperimentPaths.preload_cache_root` in `deep_lst/utils/io.py` defaults to the maintainer's + tscratch directory. For training, set `paths.preload_cache_root=` in the + command/config instead of editing Python; `paths.eozarr_root` can also override the store + root. Other entry points still use `ExperimentPaths` defaults. See `docs/CONFIGURATION.md`. The data stores (`ExperimentPaths.eozarr_root`, `/Net/Groups/BGI/work_5/scratch/WeatherGenerator/data/lowres_old`) are group-readable and stay as they are. From e8f7baa9f00d3062afa48ae769e4d5ce4be55d86 Mon Sep 17 00:00:00 2001 From: ramin_yz Date: Tue, 10 Nov 2026 21:22:40 +0100 Subject: [PATCH 3/4] fix: use the current user's preload cache directory Use DEEP_LST_CACHE_ROOT when set, otherwise choose the current user's cluster scratch path. Keep explicit configuration overrides first. Test environment defaults and override priority, including missing USER. --- deep_lst/utils/io.py | 9 ++++++++- tests/test_io.py | 31 +++++++++++++++++++++++++++++++ tests/test_training_paths.py | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/deep_lst/utils/io.py b/deep_lst/utils/io.py index e71c0fc..216a887 100644 --- a/deep_lst/utils/io.py +++ b/deep_lst/utils/io.py @@ -1,3 +1,5 @@ +import getpass +import os import re from dataclasses import dataclass, field from pathlib import Path @@ -14,7 +16,12 @@ class ExperimentPaths: eozarr_root: Path = field( default_factory=lambda: Path("/Net/Groups/BGI/work_5/scratch/WeatherGenerator/data/lowres_old") ) - preload_cache_root: Path = field(default_factory=lambda: Path("/Net/Groups/BGI/tscratch/mwessel/tmp")) + preload_cache_root: Path = field( + default_factory=lambda: Path( + os.environ.get("DEEP_LST_CACHE_ROOT") + or f"/Net/Groups/BGI/tscratch/{os.environ.get('USER') or getpass.getuser()}/tmp" + ) + ) def experiment_dir(self, experiment: str) -> Path: return self.experiment_root / experiment diff --git a/tests/test_io.py b/tests/test_io.py index c500dd2..560776b 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -18,6 +18,37 @@ def paths(tmp_path): ) +def test_cache_root_uses_environment_without_user(monkeypatch, tmp_path): + monkeypatch.setenv("DEEP_LST_CACHE_ROOT", str(tmp_path / "cache")) + monkeypatch.delenv("USER", raising=False) + assert ExperimentPaths().preload_cache_root == tmp_path / "cache" + + +@pytest.mark.parametrize("cache_root", [None, ""]) +def test_cache_root_defaults_to_current_user(monkeypatch, cache_root): + if cache_root is None: + monkeypatch.delenv("DEEP_LST_CACHE_ROOT", raising=False) + else: + monkeypatch.setenv("DEEP_LST_CACHE_ROOT", cache_root) + monkeypatch.setenv("USER", "test-user") + assert ExperimentPaths().preload_cache_root.as_posix() == "/Net/Groups/BGI/tscratch/test-user/tmp" + + +def test_cache_root_uses_account_name_when_user_is_unset(monkeypatch): + monkeypatch.delenv("DEEP_LST_CACHE_ROOT", raising=False) + monkeypatch.delenv("USER", raising=False) + monkeypatch.setattr("getpass.getuser", lambda: "test-user") + assert ExperimentPaths().preload_cache_root.as_posix() == "/Net/Groups/BGI/tscratch/test-user/tmp" + + +def test_explicit_cache_root_overrides_environment(monkeypatch, tmp_path): + monkeypatch.setenv("DEEP_LST_CACHE_ROOT", str(tmp_path / "environment-cache")) + assert ( + ExperimentPaths(preload_cache_root=tmp_path / "explicit-cache").preload_cache_root + == tmp_path / "explicit-cache" + ) + + # ── directory resolution ────────────────────────────────────────────────────── diff --git a/tests/test_training_paths.py b/tests/test_training_paths.py index b31aa0e..9e2e052 100644 --- a/tests/test_training_paths.py +++ b/tests/test_training_paths.py @@ -33,6 +33,7 @@ def test_training_passes_paths_to_data_module_and_saves_overrides(tmp_path, monk config_name="train", overrides=["wandb_logger=false", *[f"paths.{k}={v}" for k, v in overrides.items()]] ) monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DEEP_LST_CACHE_ROOT", str(tmp_path / "environment-cache")) choices = dict(model="mlp", objective="rollout", loss="smoothl1", horizon="h24", data="full") monkeypatch.setattr(train.HydraConfig, "get", lambda: SimpleNamespace(runtime=SimpleNamespace(choices=choices))) monkeypatch.setattr(train, "check_stats_transform", lambda *args: None) From 9899afca3328ad79e7edda1dd5237def4ee18a4f Mon Sep 17 00:00:00 2001 From: ramin_yz Date: Tue, 10 Nov 2026 21:22:41 +0100 Subject: [PATCH 4/4] docs: clarify where training path defaults are defined Point readers to ExperimentPaths in deep_lst/utils/io.py and the PATHS instance in dataset.py. Explain the cache fallback and override priority. --- docs/CONFIGURATION.md | 16 +++++++++++----- docs/RUNBOOK.md | 14 ++++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index b009cf9..3ab6c6d 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -59,9 +59,14 @@ be edited. ### Training data and cache paths `paths.eozarr_root` and `paths.preload_cache_root` override the data-store and preload-cache -roots used by `deep_lst/train.py`, for all three training/validation/test splits. Both default -to `null`, which keeps the existing `ExperimentPaths` defaults. Relative paths remain relative -to the repository working directory. The overrides are saved in the run's `config.yml`. +roots used by `deep_lst/train.py`, for all three training/validation/test splits. The defaults +are defined in `ExperimentPaths` in `deep_lst/utils/io.py`, also used by the `PATHS` singleton +in `deep_lst/dataset/dataset.py`. Both config values default to `null`, so these defaults apply. + +The cache default uses `DEEP_LST_CACHE_ROOT` when it is nonempty; otherwise it uses +`/Net/Groups/BGI/tscratch/$USER/tmp` (the login name is used if `USER` is unset or empty). +An explicit `paths.preload_cache_root` takes priority over this default. Relative paths remain +relative to the repository working directory. The overrides are saved in the run's `config.yml`. ``` uv run python deep_lst/train.py paths.eozarr_root=/data/eozarr paths.preload_cache_root=/scratch/deep_lst-cache @@ -69,8 +74,9 @@ uv run python deep_lst/train.py paths.eozarr_root=/data/eozarr paths.preload_cac Cache keys are unchanged; use a separate cache root when the source data content differs. -This first slice applies to training; standalone inference, statistics and fine-tuning entry -points still use their existing paths. Model, loss and data-processing settings are unchanged. +These Hydra overrides apply to training. Standalone inference, statistics and fine-tuning use +the shared `ExperimentPaths` defaults, including the cache default described above. +Model, loss and data-processing settings are unchanged. ### 2.1 Selections and splits diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index a29d56a..081272d 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -106,12 +106,14 @@ assets/statistics/_stats_.zarr z-score statistics per (modalit assets/baselines//.nc persistence / climatology / ERA5(-Land) reference forecasts ``` -Two hard-coded paths need attention in a new clone: - -1. `ExperimentPaths.preload_cache_root` in `deep_lst/utils/io.py` defaults to the maintainer's - tscratch directory. For training, set `paths.preload_cache_root=` in the - command/config instead of editing Python; `paths.eozarr_root` can also override the store - root. Other entry points still use `ExperimentPaths` defaults. See `docs/CONFIGURATION.md`. +Check these locations in a new clone: + +1. `ExperimentPaths.preload_cache_root` in `deep_lst/utils/io.py` uses `DEEP_LST_CACHE_ROOT` + when it is nonempty, otherwise `/Net/Groups/BGI/tscratch/$USER/tmp` (using the login name if + `USER` is unset or empty). For training, `paths.preload_cache_root=` on the command + line or in the config file takes priority; `paths.eozarr_root` can also override the store + root. Other entry points use the shared `ExperimentPaths` defaults, including this cache + fallback. See `docs/CONFIGURATION.md`. The data stores (`ExperimentPaths.eozarr_root`, `/Net/Groups/BGI/work_5/scratch/WeatherGenerator/data/lowres_old`) are group-readable and stay as they are.