cat_config: drop duplicate SP_v1.4.6 / SP_v1.3.6 keys, repair the survivor, guard against recurrence

cosmo_val/cat_config.yaml

@@ -153,10 +153,10 @@ SP_v1.3.6:
   ls: dashed
   marker: h
   cov_th:
-    A: 2405.3892055695346
-    n_e: 6.128201234871523
+    A: 2894.0303815287743
+    n_e: 4.002931781928292
     n_psf: 0.752316232272063
-    sigma_e: 0.379587601488189
+    sigma_e: 0.3785853255980945
   mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits
   psf:
     PSF_flag: HSM_FLAG_PSF
@@ -445,6 +445,7 @@ SP_v1.4.6:
     n_e: 5.09348802763124
     n_psf: 0.752316232272063
     sigma_e: 0.379587601488189
+  mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits
   psf:
     PSF_flag: HSM_FLAG_PSF
     PSF_size: HSM_T_PSF
@@ -461,8 +462,7 @@ SP_v1.4.6:
   shear:
     R: 1.0
     path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits
-    redshift_distr: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt
-    mask: v1.4.6/footprint_binned_8192.fits
+    redshift_path: /n17data/sguerrini/UNIONS/WL/nz/v1.4.6/nz_SP_v1.4.6_A.txt
     w_col: w_des
     e1_col: e1
     e1_col_corrected: e1_leak_corrected
@@ -519,92 +519,6 @@ SP_v1.4.6.3:
     e1_col: e1
     e2_col: e2
     path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits
-SP_v1.3.6:
-  subdir: /n17data/UNIONS/WL/v1.3.x
-  pipeline: SP
-  colour: green
-  getdist_colour: 0.0, 0.5, 1.0
-  ls: dashed
-  marker: d
-  cov_th:
-    A: 2405.3892055695346
-    n_e: 6.128201234871523
-    n_psf: 0.752316232272063
-    sigma_e: 0.379587601488189
-  mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits
-  psf:
-    PSF_flag: HSM_FLAG_PSF
-    PSF_size: HSM_T_PSF
-    star_flag: HSM_FLAG_STAR
-    star_size: HSM_T_STAR
-    hdu: 1
-    path: unions_shapepipe_psf_2022_v1.3.a.fits
-    ra_col: RA
-    dec_col: Dec
-    e1_PSF_col: HSM_G1_PSF
-    e1_star_col: HSM_G1_STAR
-    e2_PSF_col: HSM_G2_PSF
-    e2_star_col: HSM_G2_STAR
-  shear:
-    R: 1.0
-    path: v1.3.6/unions_shapepipe_cut_struc_2022_v1.3.6.fits
-    redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt
-    w_col: w_des
-    e1_col: e1
-    e1_col_corrected: e1_leak_corrected
-    e1_PSF_col: e1_PSF
-    e2_col: e2
-    e2_col_corrected: e2_leak_corrected
-    e2_PSF_col: e2_PSF
-  star:
-    ra_col: RA
-    dec_col: Dec
-    e1_col: e1
-    e2_col: e2
-    path: unions_shapepipe_star_2022_v1.0.3.fits
-SP_v1.4.6:
-  subdir: /n17data/UNIONS/WL/v1.4.x
-  pipeline: SP
-  colour: green
-  getdist_colour: 0.0, 0.5, 1.0
-  ls: dashed
-  marker: d
-  cov_th:
-    A: 2405.3892055695346
-    n_e: 6.128201234871523
-    n_psf: 0.752316232272063
-    sigma_e: 0.379587601488189
-  mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits
-  psf:
-    PSF_flag: HSM_FLAG_PSF
-    PSF_size: HSM_T_PSF
-    star_flag: HSM_FLAG_STAR
-    star_size: HSM_T_STAR
-    hdu: 1
-    path: unions_shapepipe_psf_2024_v1.4.a.fits
-    ra_col: RA
-    dec_col: Dec
-    e1_PSF_col: HSM_G1_PSF
-    e1_star_col: HSM_G1_STAR
-    e2_PSF_col: HSM_G2_PSF
-    e2_star_col: HSM_G2_STAR
-  shear:
-    R: 0.92
-    path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits
-    redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt
-    w_col: w_des
-    e1_col: e1
-    e1_col_corrected: e1_leak_corrected
-    e1_PSF_col: e1_PSF
-    e2_col: e2
-    e2_col_corrected: e2_leak_corrected
-    e2_PSF_col: e2_PSF
-  star:
-    ra_col: RA
-    dec_col: Dec
-    e1_col: e1
-    e2_col: e2
-    path: unions_shapepipe_star_2024_v1.4.a.fits
 SP_v1.4.6.3_B:
   subdir: /n17data/UNIONS/WL/v1.4.x
   pipeline: SP

src/sp_validation/tests/test_cat_config_no_duplicate_keys.py

@@ -0,0 +1,72 @@
+"""Guard: ``cosmo_val/cat_config.yaml`` has no repeated mapping keys.
+
+PyYAML accepts duplicate keys silently, keeping the last occurrence. A merge
+that resolves a conflict by keeping both sides therefore produces a config
+where the *shadowed* block is invisible to every consumer and to every test
+that loads it with ``yaml.safe_load`` -- which is how two ``SP_v1.4.6`` and
+two ``SP_v1.3.6`` entries lived on ``develop`` with the stale ones winning.
+
+This loads the catalogue config with a loader that raises instead.
+
+:Author: cdaley
+
+"""
+
+from pathlib import Path
+
+import pytest
+import yaml
+
+pytestmark = pytest.mark.fast
+
+
+class DuplicateKeyError(ValueError):
+    """Raised when a YAML mapping repeats a key."""
+
+
+class UniqueKeySafeLoader(yaml.SafeLoader):
+    """``yaml.SafeLoader`` that rejects repeated keys instead of overwriting."""
+
+    def construct_mapping(self, node, deep=False):
+        mapping = {}
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if key in mapping:
+                raise DuplicateKeyError(
+                    f"duplicate key {key!r} at line {key_node.start_mark.line + 1} "
+                    f"(first seen at line {mapping[key] + 1})"
+                )
+            mapping[key] = key_node.start_mark.line
+        return super().construct_mapping(node, deep=deep)
+
+
+def _repo_root() -> Path:
+    for parent in Path(__file__).resolve().parents:
+        if (parent / "pyproject.toml").exists():
+            return parent
+    raise RuntimeError("could not locate repo root (no pyproject.toml above test)")
+
+
+def load_unique(path: Path):
+    """Parse ``path`` as YAML, raising ``DuplicateKeyError`` on repeated keys."""
+    with Path(path).open() as handle:
+        return yaml.load(handle, Loader=UniqueKeySafeLoader)
+
+
+def test_loader_rejects_duplicate_keys(tmp_path):
+    """The loader itself catches a repeated key (and ``safe_load`` does not)."""
+    config = tmp_path / "dup.yaml"
+    config.write_text("a:\n  x: 1\nb:\n  y: 2\na:\n  x: 3\n")
+
+    assert yaml.safe_load(config.read_text()) == {"a": {"x": 3}, "b": {"y": 2}}
+    with pytest.raises(DuplicateKeyError, match="duplicate key 'a'"):
+        load_unique(config)
+
+
+def test_cat_config_has_no_duplicate_keys():
+    """``cosmo_val/cat_config.yaml`` parses with no key shadowing another."""
+    config_path = _repo_root() / "cosmo_val" / "cat_config.yaml"
+    assert config_path.exists(), f"missing config: {config_path}"
+
+    config = load_unique(config_path)
+    assert config, "cat_config.yaml parsed empty"