From 0534daf6cfd1407dc30204a1dee1fa4dfcc0e436 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 30 Jun 2026 10:18:44 +0200 Subject: [PATCH 1/8] ngmix: add config-selectable UberSeg neighbour masking (BLEND_HANDLING) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new ngmix-module option BLEND_HANDLING = {noisefill, uberseg} selects how a neighbour sharing a galaxy's stamp is treated. The default `noisefill` is byte-for-byte the historical behaviour (flagged pixels replaced by a noise realisation, kept at inverse-variance weight); `uberseg` instead hard-masks (weight -> 0) every pixel closer to a neighbour's segmentation footprint than to the central object's, leaving the image untouched. `uberseg_weight()` reimplements Erin Sheldon's MEDS uberseg (the `meds._uberseg.uberseg_tree` nearest-segment Voronoi partition) with a scipy cKDTree, rather than depending on `meds` whose import drags the full fitsio/esutil stack. The surviving central core is a single connected, roughly circular region — emergent geometry of the partition, not a separate aperture, faithful to MEDS `get_uberseg`. Plumbed runner -> Ngmix -> do_ngmix_metacal -> make_ngmix_observation -> prepare_ngmix_weights; Postage_stamp gains a per-epoch `segs` list. The segmentation-map *source* is not wired here (it is plumbing-gated: the seg map is the coadd's, ngmix fits per-epoch stamps); selecting `uberseg` without a seg map raises with a pointer to the gating issue. Tests: synthetic two-object stamp asserts the mask geometry (neighbour-side zeroed, central core single-connected, matches brute-force nearest-segment), the noisefill default is byte-identical to the legacy noise-fill, and uberseg leaves the image untouched while zeroing neighbour/flagged weight. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018DLfuwQzavQQ9GbcKZQPc5 --- src/shapepipe/modules/ngmix_package/ngmix.py | 144 +++++++++++++- src/shapepipe/modules/ngmix_runner.py | 10 + tests/module/test_ngmix_uberseg.py | 198 +++++++++++++++++++ 3 files changed, 347 insertions(+), 5 deletions(-) create mode 100644 tests/module/test_ngmix_uberseg.py diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 386c25efc..82bca3d63 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -14,10 +14,14 @@ from astropy.io import fits from modopt.math.stats import sigma_mad from ngmix.observation import Observation, ObsList +from scipy.spatial import cKDTree from sqlitedict import SqliteDict from shapepipe.pipeline import file_io +# Neighbour treatments selectable with the BLEND_HANDLING option. +BLEND_HANDLINGS = ("noisefill", "uberseg") + # Gaussian half-light radius per unit sigma: r50 = sqrt(2 ln 2) * sigma # = 1.17741 * sigma. With the ngmix area parameter T = 2 sigma^2 this # gives r50 = SIGMA_TO_R50 * sqrt(T / 2) = sqrt(ln 2 * T). @@ -149,6 +153,9 @@ def __init__( self.weights = [] self.flags = [] self.bkg_rms = [] + # Per-epoch segmentation stamps (object NUMBERs), used only by the + # "uberseg" blend handling; empty for the default noise-fill path. + self.segs = [] self.jacobs = [] # Per-epoch full WCS and the object's sky position, used only by the # "wcs" centroid source (skipped for the default "hsm" path). @@ -263,6 +270,7 @@ def __init__( id_obj_min=-1, id_obj_max=-1, centroid_source="hsm", + blend_handling="noisefill", ): if len(input_file_list) not in {6, 7}: @@ -271,6 +279,12 @@ def __init__( + " required is 6 or 7" ) + if blend_handling not in BLEND_HANDLINGS: + raise ValueError( + f"Unknown BLEND_HANDLING '{blend_handling}'; expected one of" + + f" {BLEND_HANDLINGS}" + ) + self._tile_cat_path = input_file_list[0] self._vignet_cat = Vignet( input_file_list[1], @@ -294,6 +308,7 @@ def __init__( self._id_obj_min = id_obj_min self._id_obj_max = id_obj_max self._centroid_source = centroid_source + self._blend_handling = blend_handling self._w_log = w_log @@ -668,6 +683,8 @@ def process(self): flux_guess, self._rng, centroid_source=self._centroid_source, + blend_handling=self._blend_handling, + object_number=obj_id, ) except Exception as ee: self._w_log.info( @@ -957,7 +974,65 @@ def get_noise(gal, weight, guess, pixel_scale, thresh=1.2): return sig_noise -def prepare_ngmix_weights(gal, weight, flag, rng, bkg_rms=None): +def uberseg_weight(weight, seg, object_number): + """Zero a stamp's weight on neighbour-side pixels (Sheldon/MEDS UberSeg). + + Each pixel is assigned to the object whose segmentation footprint it is + nearest to — a nearest-segment Voronoi partition — and only pixels + assigned to the central object keep their weight. This is the algorithm + behind ``meds.MEDS.get_uberseg`` / ``meds._uberseg.uberseg_tree``, + reimplemented here (the C extension lives in ``meds``, whose import drags + the full ``fitsio``/``esutil`` stack) with a ``cKDTree`` nearest-neighbour + query in place of the C k-d tree. + + Because the partition is by distance to the nearest footprint, the pixels + surviving around a compact central object form a single connected, + roughly circular core; the "circularisation" is emergent geometry, not a + separate aperture. Unlike the noise-fill treatment, masked pixels are + handed to ngmix as a hard mask (weight = 0), never replaced by a noise + realisation. + + Parameters + ---------- + weight : numpy.ndarray + Per-pixel weight (inverse variance) map for the stamp. + seg : numpy.ndarray + Segmentation map on the same grid as ``weight``: 0 for sky, the + SExtractor object number for each detected object's footprint. + object_number : int + Segmentation label of the central object (its SExtractor ``NUMBER``). + + Returns + ------- + numpy.ndarray + Copy of ``weight`` with neighbour-side pixels zeroed. + """ + weight = np.copy(weight) + + obj_pix = np.argwhere(seg != 0) + # No detected footprint, or only sky plus the central object: there is + # no neighbour to mask, so the weight is unchanged (cf. MEDS' early + # ``len(np.unique(seg)) == 2`` return). + if obj_pix.shape[0] == 0: + return weight + labels = seg[seg != 0] + if np.all(labels == object_number): + return weight + + # Nearest segmentation pixel for every stamp pixel; zero the weight + # wherever that nearest footprint belongs to a neighbour rather than to + # the central object. + grid = np.indices(seg.shape).reshape(2, -1).T + _, nearest = cKDTree(obj_pix).query(grid) + nearest_label = labels[nearest].reshape(seg.shape) + weight[nearest_label != object_number] = 0.0 + + return weight + +def prepare_ngmix_weights( + gal, weight, flag, rng, bkg_rms=None, + blend_handling="noisefill", seg=None, object_number=None, +): """bookkeeping for ngmix weights. runs on a single galaxy and epoch pixel scale and galaxy guess TO DO: decide if we want galaxy guess stuff @@ -973,16 +1048,36 @@ def prepare_ngmix_weights(gal, weight, flag, rng, bkg_rms=None): bkg_rms : numpy.ndarray, optional Per-pixel background RMS map. If supplied, unmasked pixels use ``1 / bkg_rms**2`` as the ngmix inverse variance. + blend_handling : {"noisefill", "uberseg"}, optional + How to treat pixels shared with a neighbour. ``"noisefill"`` (default) + replaces flagged pixels with a noise realisation and keeps their + inverse-variance weight — the historical behaviour. ``"uberseg"`` + instead hard-masks (weight = 0) every pixel closer to a neighbour's + segmentation footprint than to the central object's, leaving the + image untouched (see :func:`uberseg_weight`). + seg : numpy.ndarray, optional + Segmentation map on the stamp grid (object NUMBERs). Required for + ``blend_handling="uberseg"``; ignored otherwise. + object_number : int, optional + Central object's segmentation label. Required for + ``blend_handling="uberseg"``; ignored otherwise. Returns ------- numpy.ndarray - Galaxy image with masked pixels replaced by noise. + Galaxy image. For ``"noisefill"`` masked pixels are replaced by noise; + for ``"uberseg"`` the image is returned untouched. numpy.ndarray Variance map for NGMIX. numpy.ndarray Noise image. """ + if blend_handling not in BLEND_HANDLINGS: + raise ValueError( + f"Unknown blend_handling '{blend_handling}'; expected one of" + + f" {BLEND_HANDLINGS}" + ) + mask = np.copy(weight) != 0 mask[flag != 0] = False @@ -1015,7 +1110,21 @@ def prepare_ngmix_weights(gal, weight, flag, rng, bkg_rms=None): noise_img_gal = rng.standard_normal(gal.shape) * sig_noise gal_masked = np.copy(gal) - if (~mask).any(): + if blend_handling == "uberseg": + # Hard-mask neighbour-side pixels (weight -> 0) from the segmentation + # geometry; the image is left untouched (the masked pixels carry no + # weight, so ngmix ignores them in the likelihood). Bad/flagged + # pixels already sit at weight 0 from the mask above. + if seg is None or object_number is None: + raise ValueError( + "blend_handling='uberseg' requires a segmentation map and the" + + " central object_number; none reached prepare_ngmix_weights." + + " The seg-map source is gated (see CosmoStat/shapepipe issue" + + " on UberSeg segmentation plumbing)." + ) + weight_map = uberseg_weight(weight_map, seg, object_number) + elif (~mask).any(): + # noisefill (default): replace masked pixels with a noise realisation. gal_masked[~mask] = noise_img_gal[~mask] return gal_masked, weight_map, noise_img @@ -1023,6 +1132,7 @@ def prepare_ngmix_weights(gal, weight, flag, rng, bkg_rms=None): def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="hsm", wcs_full=None, ra=None, dec=None, + blend_handling="noisefill", seg=None, object_number=None, ): """Build an ngmix Observation for a single galaxy epoch. @@ -1061,6 +1171,15 @@ def make_ngmix_observation( ra, dec : float, optional Object sky position in degrees. Required for ``centroid_source="wcs"`` (ignored for ``"hsm"``). + blend_handling : {"noisefill", "uberseg"}, optional + Neighbour treatment passed through to :func:`prepare_ngmix_weights`; + the default ``"noisefill"`` is the historical behaviour. + seg : numpy.ndarray, optional + Segmentation map on the stamp grid. Required for + ``blend_handling="uberseg"`` (ignored otherwise). + object_number : int, optional + Central object's segmentation label. Required for + ``blend_handling="uberseg"`` (ignored otherwise). Returns ------- @@ -1074,7 +1193,8 @@ def make_ngmix_observation( psf_obs = Observation(psf, jacobian=psf_jacob) gal_masked, weight_map, noise_img = prepare_ngmix_weights( - gal, weight, flag, rng, bkg_rms=bkg_rms + gal, weight, flag, rng, bkg_rms=bkg_rms, + blend_handling=blend_handling, seg=seg, object_number=object_number, ) if centroid_source == "hsm": @@ -1207,7 +1327,10 @@ def make_runners(prior, flux_guess, rng): ) -def do_ngmix_metacal(stamp, prior, flux_guess, rng, centroid_source="hsm"): +def do_ngmix_metacal( + stamp, prior, flux_guess, rng, centroid_source="hsm", + blend_handling="noisefill", object_number=None, +): """Do Ngmix Metacal. Performs metacalibration on a single multi-epoch object and returns the @@ -1229,6 +1352,14 @@ def do_ngmix_metacal(stamp, prior, flux_guess, rng, centroid_source="hsm"): adaptive-moment centroid); ``"wcs"`` uses the catalog sky position projected through the WCS — see that function for the star-vs-galaxy rationale. + blend_handling : {"noisefill", "uberseg"}, optional + Neighbour treatment passed through to + :func:`make_ngmix_observation`; the default ``"noisefill"`` is the + historical behaviour. ``"uberseg"`` consumes ``stamp.segs`` and + ``object_number``. + object_number : int, optional + Central object's segmentation label (its SExtractor ``NUMBER``). + Required for ``blend_handling="uberseg"`` (ignored otherwise). Returns ------- @@ -1255,6 +1386,9 @@ def do_ngmix_metacal(stamp, prior, flux_guess, rng, centroid_source="hsm"): wcs_full=stamp.wcs[n_e] if n_e < len(stamp.wcs) else None, ra=stamp.ra[n_e] if n_e < len(stamp.ra) else None, dec=stamp.dec[n_e] if n_e < len(stamp.dec) else None, + blend_handling=blend_handling, + seg=stamp.segs[n_e] if n_e < len(stamp.segs) else None, + object_number=object_number, ) gal_obs_list.append(gal_obs) diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 31028c7f8..378a72a78 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -88,6 +88,15 @@ def ngmix_runner( else: centroid_source = "wcs" + # Neighbour treatment: "noisefill" (default, historical) replaces a + # neighbour's pixels with a noise realisation; "uberseg" hard-masks + # (weight -> 0) every pixel closer to a neighbour than to the central + # object, from the segmentation map. See the ngmix module docstrings. + if config.has_option(module_config_sec, "BLEND_HANDLING"): + blend_handling = config.get(module_config_sec, "BLEND_HANDLING") + else: + blend_handling = "noisefill" + # Initialise class instance ngmix_inst = Ngmix( input_file_list, @@ -101,6 +110,7 @@ def ngmix_runner( id_obj_min=id_obj_min, id_obj_max=id_obj_max, centroid_source=centroid_source, + blend_handling=blend_handling, ) # Process ngmix shape measurement and metacalibration diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py new file mode 100644 index 000000000..98a586d73 --- /dev/null +++ b/tests/module/test_ngmix_uberseg.py @@ -0,0 +1,198 @@ +"""UberSeg neighbour-masking unit tests. + +Covers the ``BLEND_HANDLING = uberseg`` option added to the ngmix module: + +* :func:`uberseg_weight` — the Sheldon/MEDS nearest-segment Voronoi mask. + Geometry assertions on a synthetic two-object stamp: neighbour-side pixels + are zeroed, the surviving central core is a *single connected* region (the + emergent "circularisation"), and the neighbour footprint is fully removed. +* :func:`prepare_ngmix_weights` — the ``noisefill`` default is byte-for-byte + unchanged (asserted against an independent recomputation of the legacy + three-line noise-fill on a shared RNG), while ``uberseg`` hard-masks the + weight (weight -> 0) and leaves the image untouched. +* The error contract when ``uberseg`` is selected without a segmentation map + (the seg-map source is plumbing-gated upstream). +""" + +import numpy as np +import numpy.testing as npt +import pytest +from scipy import ndimage + +from shapepipe.modules.ngmix_package.ngmix import ( + prepare_ngmix_weights, + uberseg_weight, +) + + +def two_object_seg(npix=41, sep=12, r_central=3, r_neighbour=3): + """Stamp seg map: central object (label 1) at centre, neighbour (label 2) + offset ``sep`` pixels along the column axis. Returns (seg, centre, neigh). + """ + seg = np.zeros((npix, npix), dtype=np.int32) + yy, xx = np.indices((npix, npix)) + c = npix // 2 + centre = (c, c) + neigh = (c, c + sep) + seg[(yy - centre[0]) ** 2 + (xx - centre[1]) ** 2 <= r_central ** 2] = 1 + seg[(yy - neigh[0]) ** 2 + (xx - neigh[1]) ** 2 <= r_neighbour ** 2] = 2 + return seg, centre, neigh + + +def test_uberseg_zeros_neighbour_side_keeps_centre(): + """Neighbour-side pixels lose their weight; the central pixel keeps it.""" + seg, centre, neigh = two_object_seg() + weight = np.ones_like(seg, dtype=float) + + out = uberseg_weight(weight, seg, object_number=1) + + # Central object pixel kept; deep-neighbour pixel zeroed. + assert out[centre] == 1.0 + assert out[neigh] == 0.0 + # Every neighbour-footprint pixel is removed from the fit. + assert np.all(out[seg == 2] == 0.0) + # Every central-footprint pixel survives. + assert np.all(out[seg == 1] == 1.0) + # The input weight is not mutated in place. + assert np.all(weight == 1.0) + + +def test_uberseg_core_is_single_connected_region(): + """The surviving (kept-weight) region is one connected component — the + emergent circular core of the nearest-segment Voronoi partition.""" + seg, _, _ = two_object_seg() + weight = np.ones_like(seg, dtype=float) + + out = uberseg_weight(weight, seg, object_number=1) + + kept = out > 0 + _, n_components = ndimage.label(kept) + assert n_components == 1 + # The partition splits the stamp: some pixels survive, some are masked. + assert kept.any() and (~kept).any() + + +def test_uberseg_partition_is_the_perpendicular_bisector(): + """With the neighbour due-right of the centre, the mask is the far-right + half-plane beyond the footprint bisector: the left edge survives, the + column past the neighbour is gone.""" + seg, centre, neigh = two_object_seg(npix=41, sep=12) + weight = np.ones_like(seg, dtype=float) + + out = uberseg_weight(weight, seg, object_number=1) + + c_row, c_col = centre + assert out[c_row, 0] == 1.0 # far side from the neighbour: kept + assert out[c_row, -1] == 0.0 # neighbour side edge: masked + + +def test_uberseg_no_neighbour_is_passthrough(): + """A stamp with only the central object (or empty seg) is unchanged.""" + npix = 21 + weight = np.random.default_rng(0).random((npix, npix)) + 0.1 + + # Only the central object present. + seg = np.zeros((npix, npix), dtype=np.int32) + seg[8:13, 8:13] = 1 + npt.assert_array_equal(uberseg_weight(weight, seg, object_number=1), weight) + + # Wholly empty seg (no detections). + empty = np.zeros((npix, npix), dtype=np.int32) + npt.assert_array_equal(uberseg_weight(weight, empty, object_number=1), weight) + + +def test_uberseg_matches_bruteforce_nearest_segment(): + """The cKDTree result equals the O(N^2) brute-force nearest-segment rule + (Sheldon's non-C fallback) it stands in for.""" + seg, _, _ = two_object_seg(npix=25, sep=8) + weight = np.ones_like(seg, dtype=float) + + out = uberseg_weight(weight, seg, object_number=1) + + obj = np.argwhere(seg != 0) + labels = seg[seg != 0] + brute = np.ones_like(weight) + for i in range(seg.shape[0]): + for j in range(seg.shape[1]): + d2 = (i - obj[:, 0]) ** 2 + (j - obj[:, 1]) ** 2 + if labels[np.argmin(d2)] != 1: + brute[i, j] = 0.0 + npt.assert_array_equal(out, brute) + + +# --- prepare_ngmix_weights: default unchanged, uberseg hard-masks ---------- + +def _gal_flag_weight(npix=41, seed=7): + rng = np.random.default_rng(seed) + gal = rng.normal(0.0, 3.0, (npix, npix)) + 50.0 + weight = np.ones((npix, npix)) + flag = np.zeros((npix, npix), dtype=np.int32) + # A handful of flagged (bad) pixels — cosmic-ray-like. + flag[5, 5] = 1 + flag[30, 12] = 2 + return gal, flag, weight + + +def test_noisefill_default_is_byte_identical_to_legacy(): + """The default path reproduces the legacy three-line noise-fill exactly + (same RNG stream): masked pixels replaced by noise, weight 1/sigma^2.""" + gal, flag, weight = _gal_flag_weight() + + gal_out, w_out, noise_out = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(123), + ) + + # Independent recomputation of the legacy algorithm on the same stream. + from modopt.math.stats import sigma_mad + rng = np.random.RandomState(123) + mask = np.copy(weight) != 0 + mask[flag != 0] = False + sig = sigma_mad(gal) + w_exp = mask.astype(float) / sig ** 2 + noise_exp = rng.standard_normal(gal.shape) * sig + noise_gal = rng.standard_normal(gal.shape) * sig + gal_exp = np.copy(gal) + gal_exp[~mask] = noise_gal[~mask] + + npt.assert_array_equal(gal_out, gal_exp) + npt.assert_array_equal(w_out, w_exp) + npt.assert_array_equal(noise_out, noise_exp) + + +def test_uberseg_hard_masks_weight_and_leaves_image_untouched(): + """uberseg: image returned untouched, weight zeroed on neighbour-side and + flagged pixels, positive on the central core.""" + npix = 41 + gal, flag, weight = _gal_flag_weight(npix=npix) + seg, centre, neigh = two_object_seg(npix=npix, sep=12) + + gal_out, w_out, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(5), + blend_handling="uberseg", seg=seg, object_number=1, + ) + + # Image untouched under uberseg (no noise fill). + npt.assert_array_equal(gal_out, gal) + # Neighbour footprint hard-masked; central centre kept. + assert np.all(w_out[seg == 2] == 0.0) + assert w_out[centre] > 0.0 + # Flagged bad pixels remain at weight 0 (folded into the base mask). + assert w_out[5, 5] == 0.0 + + +def test_uberseg_requires_seg_and_object_number(): + gal, flag, weight = _gal_flag_weight() + with pytest.raises(ValueError, match="requires a segmentation map"): + prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), + blend_handling="uberseg", + ) + + +def test_unknown_blend_handling_raises(): + gal, flag, weight = _gal_flag_weight() + with pytest.raises(ValueError, match="Unknown blend_handling"): + prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), + blend_handling="bogus", + ) From d35e446c13bcc2d827f5812648d4aa286b019290 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 30 Jun 2026 23:18:13 +0200 Subject: [PATCH 2/8] docs(ngmix): credit Erin Sheldon for UberSeg; correct the reimplement rationale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Name Erin Sheldon, the esheldon/meds repo, the MacCrann/Zuntz lineage, and DES use in the uberseg_weight docstring, and state that the reimplementation was validated bit-for-bit against his reference get_uberseg. Correct the dependency note: fitsio is already in the ShapePipe stack; esutil (+ a C-extension build) is what import meds would add. The argument for reimplementing is proportionality (a 5-line scipy cKDTree partition vs the whole MEDS file-format library), now backed by the validation harness in the uberseg fiber. Docstring-only; no behaviour change. Local checkpoint — not pushed (pending the next session's read of Axel's comment). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01K35XrdS1Bn8EKp4VJGYpp6 --- src/shapepipe/modules/ngmix_package/ngmix.py | 31 ++++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 82bca3d63..0274935bf 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -975,15 +975,28 @@ def get_noise(gal, weight, guess, pixel_scale, thresh=1.2): return sig_noise def uberseg_weight(weight, seg, object_number): - """Zero a stamp's weight on neighbour-side pixels (Sheldon/MEDS UberSeg). - - Each pixel is assigned to the object whose segmentation footprint it is - nearest to — a nearest-segment Voronoi partition — and only pixels - assigned to the central object keep their weight. This is the algorithm - behind ``meds.MEDS.get_uberseg`` / ``meds._uberseg.uberseg_tree``, - reimplemented here (the C extension lives in ``meds``, whose import drags - the full ``fitsio``/``esutil`` stack) with a ``cKDTree`` nearest-neighbour - query in place of the C k-d tree. + """Zero a stamp's weight on neighbour-side pixels — the UberSeg treatment. + + UberSeg is Erin Sheldon's MEDS/ngmix neighbour mask (``esheldon/meds``, + https://github.com/esheldon/meds — ``MEDS.get_uberseg`` / + ``meds._uberseg.uberseg_tree``, in MEDS itself "adapted from Niall MacCrann + and Joe Zuntz", and used in the DES shear pipeline). Each stamp pixel is + assigned to the object whose segmentation footprint it lies nearest to — a + nearest-segment Voronoi partition — and only pixels assigned to the central + object keep their weight. + + This function **reimplements** the partition rather than depending on + ``meds``: it is a five-line ``scipy.spatial.cKDTree`` nearest-neighbour + query, and pulling in the whole MEDS-file library — plus ``esutil`` and a + C-extension build (``fitsio`` is already in the ShapePipe stack; ``esutil`` + is not) — for one standard geometric operation is disproportionate. The + reimplementation was validated bit-for-bit against Sheldon's own reference + ``get_uberseg`` over a battery of synthetic and random multi-object seg + maps: the two masks agree on every pixel except exact Voronoi-boundary ties + (a measure-zero tie-break convention — ``argmin`` first-min vs cKDTree + order — scientifically inert). See the ``uberseg`` fiber's ``validation/`` + harness for the equivalence proof and figures. The cKDTree stands in for + the C k-d tree; results are identical up to that boundary convention. Because the partition is by distance to the nearest footprint, the pixels surviving around a compact central object form a single connected, From 35fc2d5eee45aa6039f7042b8304284c57013337 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Jul 2026 22:41:25 +0200 Subject: [PATCH 3/8] ngmix: coadd-seg UberSeg plumbing, centre-pixel central ID, neighbour flag Populate the UberSeg segmentation transport end to end, closing the gap where selecting BLEND_HANDLING=uberseg raised for lack of a seg source (shapepipe#776). - Tile_cat loads a coadd-frame segmentation VIGNET catalogue (seg_cat_path), one integer stamp per object, row-aligned to the tile catalogue. It is overlaid unchanged on every epoch (no per-epoch reprojection): prepare_ postage_stamps copies the coadd seg onto each surviving epoch, MegaCam- flipped to match its galaxy stamp so the overlay stays registered. - central_seg_label identifies the central object from the seg stamp's centre pixel rather than a plumbed SExtractor NUMBER, keeping the stamp self- describing and robust to a few-pixel coadd-vs-epoch offset. A sky centre pixel fails fast; the object is dropped by the per-object try/except. - uberseg_weight gains dilate_neighbour: binary-dilation iterations enlarging the neighbour mask to absorb that offset. Additive over the validated Voronoi partition; dilate_neighbour=0 recovers the pure Sheldon mask. - seg_has_neighbour computes a per-object blend flag (non-central footprint present), carried through compile_results into every shear HDU. - Ngmix fails fast at construction when uberseg is selected without seg_cat_path, replacing the deep per-epoch error as the primary guard. The default noisefill path is byte-for-byte unchanged; every new argument is optional and ignored off the uberseg branch. Co-Authored-By: Claude Fable 5 --- src/shapepipe/modules/ngmix_package/ngmix.py | 214 +++++++++++++++++-- 1 file changed, 200 insertions(+), 14 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index b0c29c21f..939243026 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -17,6 +17,7 @@ from cs_util import size as cs_size from modopt.math.stats import sigma_mad from ngmix.observation import Observation, ObsList +from scipy.ndimage import binary_dilation from scipy.spatial import cKDTree from sqlitedict import SqliteDict @@ -130,14 +131,23 @@ class Tile_cat(): Parameters ---------- - cat_path + cat_path : str + Path to the tile SExtractor catalogue. + seg_cat_path : str, optional + Path to the coadd-frame segmentation VIGNET catalogue (a CLASSIC-mode + vignetmaker output cut from the tile ``SEGMENTATION`` check image), + row-aligned to ``cat_path``. When given, ``self.seg`` holds one integer + seg stamp per object for the ``"uberseg"`` blend handling; ``None`` + leaves ``self.seg`` unset (the noise-fill path is unaffected). """ def __init__( self, - cat_path + cat_path, + seg_cat_path=None, ): self.cat_path = cat_path + self.seg_cat_path = seg_cat_path if cat_path: self.get_data(cat_path) @@ -160,6 +170,20 @@ def get_data(self, cat_path): tile_cat.close() + # Coadd-frame SExtractor segmentation stamp (integer labels), one per + # object and row-aligned to the tile catalogue, overlaid unchanged on + # every epoch for uberseg neighbour masking (shapepipe#776). None -> + # uberseg unavailable; the noise-fill path is unaffected. + self.seg = None + if self.seg_cat_path: + seg_cat = file_io.FITSCatalogue( + self.seg_cat_path, + SEx_catalogue=True, + ) + seg_cat.open() + self.seg = np.copy(seg_cat.get_data()['VIGNET']) + seg_cat.close() + class Postage_stamp(): """Galaxy Postage Stamp. @@ -184,8 +208,11 @@ def __init__( self.weights = [] self.flags = [] self.bkg_rms = [] - # Per-epoch segmentation stamps (object NUMBERs), used only by the - # "uberseg" blend handling; empty for the default noise-fill path. + # Segmentation stamps, one per epoch, used only by the "uberseg" blend + # handling; empty for the default noise-fill path. All epochs carry the + # SAME coadd-frame seg stamp (shapepipe#776: one coadd seg per object, + # no per-epoch reprojection), each MegaCam-flipped to match its galaxy + # stamp so the overlay stays registered. self.segs = [] self.jacobs = [] # Per-epoch full WCS and the object's sky position, used only by the @@ -280,11 +307,24 @@ class Ngmix(object): (robust for galaxies); ``"wcs"`` uses the catalog sky position projected through the WCS (better for stars, whose HSM moments are noisy). See :func:`make_ngmix_observation`. + blend_handling : {"noisefill", "uberseg"}, optional + Neighbour treatment; ``"noisefill"`` (default) is the historical + noise-fill, ``"uberseg"`` hard-masks neighbour-side pixels from the + coadd segmentation map and requires ``seg_cat_path``. + seg_cat_path : str, optional + Path to the coadd-frame segmentation VIGNET catalogue (see + :class:`Tile_cat`). Required when ``blend_handling="uberseg"``. + dilate_neighbour : int, optional + Neighbour-mask dilation iterations for ``"uberseg"`` (see + :func:`uberseg_weight`); the default is ``1``. Raises ------ IndexError If the length of the input file list is incorrect + ValueError + If ``blend_handling`` is unknown, or ``"uberseg"`` is selected without + ``seg_cat_path``. """ @@ -302,6 +342,8 @@ def __init__( id_obj_max=-1, centroid_source="hsm", blend_handling="noisefill", + seg_cat_path=None, + dilate_neighbour=1, ): if len(input_file_list) not in {6, 7}: @@ -316,6 +358,14 @@ def __init__( + f" {BLEND_HANDLINGS}" ) + # Fail fast at construction (not deep in the per-epoch loop) when + # uberseg is requested without its segmentation input (shapepipe#776). + if blend_handling == "uberseg" and seg_cat_path is None: + raise ValueError( + "blend_handling='uberseg' requires SEG_VIGNET_PATH (the coadd" + + " SExtractor segmentation vignets); none configured." + ) + self._tile_cat_path = input_file_list[0] self._vignet_cat = Vignet( input_file_list[1], @@ -340,6 +390,8 @@ def __init__( self._id_obj_max = id_obj_max self._centroid_source = centroid_source self._blend_handling = blend_handling + self._seg_cat_path = seg_cat_path + self._dilate_neighbour = dilate_neighbour self._w_log = w_log @@ -431,6 +483,7 @@ def compile_results(self, results): 'id', 'n_epoch_model', 'mcal_types_fail', + 'neighbour_flag', 'nfev_fit', # galaxy 'g1', @@ -490,6 +543,11 @@ def compile_results(self, results): output_dict[name]["mcal_types_fail"].append( results[idx]["mcal_types_fail"] ) + # Per-object blend flag (see process()); replicated across all + # shear types like id / n_epoch_model / mcal_types_fail. + output_dict[name]["neighbour_flag"].append( + results[idx]["neighbour_flag"] + ) # ngmix 2.x reports the solver's function-evaluation count # (nfev, ~tens-hundreds; -1 on some failures), not the v1 # 1-5 retry count, so the column is named accordingly. Fits @@ -665,7 +723,7 @@ def process(self): Dictionary containing the NGMIX metacal results """ - tile_cat = Tile_cat(self._tile_cat_path) + tile_cat = Tile_cat(self._tile_cat_path, self._seg_cat_path) vignet_cat = self._vignet_cat final_res = [] @@ -709,6 +767,14 @@ def process(self): if tile_cat.flux is not None else 1.0 ) + # Central object's seg label from the centre pixel (uberseg), + # else the SExtractor NUMBER (noisefill ignores it). Fails loud + # on a mis-centred seg stamp — the object is then dropped below. + central_label = ( + central_seg_label(tile_cat.seg[i_tile]) + if self._blend_handling == "uberseg" + else obj_id + ) res, psf_res, psf_orig_res = do_ngmix_metacal( stamp, prior, @@ -716,7 +782,8 @@ def process(self): self._rng, centroid_source=self._centroid_source, blend_handling=self._blend_handling, - object_number=obj_id, + object_number=central_label, + dilate_neighbour=self._dilate_neighbour, ) except Exception as ee: self._w_log.info( @@ -726,6 +793,14 @@ def process(self): continue res['obj_id'] = obj_id + # Neighbour flag: does the coadd seg stamp hold any non-central, + # non-zero label? Systematics-test hook (shapepipe#776); computed + # from the raw coadd seg, 0 when uberseg is off / seg absent. + res['neighbour_flag'] = int( + seg_has_neighbour(tile_cat.seg[i_tile], central_label) + if getattr(tile_cat, "seg", None) is not None + else 0 + ) # epochs that survived the PSF fit and entered the model, # not the number of epochs submitted (v1 contract) res['n_epoch_model'] = psf_res['n_epoch'] @@ -824,6 +899,17 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat): if stamp.megacam_flip and tile_vign is not None: tile_vign = Ngmix.MegaCamFlip(tile_vign, int(ccd_n)) + # Coadd-frame segmentation stamp for this object, MegaCam-flipped to + # match the (flipped) galaxy stamp so the overlay stays registered. + # The SAME coadd seg rides every epoch (shapepipe#776: no reprojection). + tile_seg = ( + np.copy(tile_cat.seg[i_tile]) + if getattr(tile_cat, "seg", None) is not None + else None + ) + if stamp.megacam_flip and tile_seg is not None: + tile_seg = Ngmix.MegaCamFlip(tile_seg, int(ccd_n)) + flag_vign = ( vignet.flag_vign_cat[str(obj_id)][expccd_name]['VIGNET'] ) @@ -872,6 +958,8 @@ def prepare_postage_stamps(vignet, obj_id, i_tile, tile_cat): stamp.weights.append(weight_vign_scaled) stamp.flags.append(flag_vign) stamp.bkg_rms.append(bkg_rms_vign_scaled) + if tile_seg is not None: + stamp.segs.append(tile_seg) stamp.jacobs.append(jacob) # For the "wcs" centroid source (see make_ngmix_observation). stamp.wcs.append(epoch_wcs) @@ -1007,7 +1095,68 @@ def get_noise(gal, weight, guess, pixel_scale, thresh=1.2): return sig_noise -def uberseg_weight(weight, seg, object_number): +def central_seg_label(seg): + """Central object's label — the value at the seg stamp's centre pixel. + + The central object is identified geometrically, from the centre pixel of + the coadd segmentation stamp, rather than by plumbing the SExtractor + ``NUMBER`` through the pipeline (shapepipe#776). The stamp is cut centred + on the object, so its central pixel lies in that object's footprint; this + keeps the seg stamp self-describing and robust to a few-pixel coadd-vs- + epoch registration offset that could otherwise mismatch a plumbed NUMBER. + + Parameters + ---------- + seg : numpy.ndarray + Segmentation stamp (integer SExtractor labels; 0 for sky). + + Returns + ------- + int + Label at the centre pixel — the central object's segmentation id. + + Raises + ------ + ValueError + If the centre pixel is sky (label 0): the cutout is not centred on any + detected footprint (bad centroid / dropped detection), so uberseg + cannot define a central object. Fail fast and loud; the object is then + dropped by the per-object ``try/except`` in :meth:`Ngmix.process`. + """ + cy, cx = seg.shape[0] // 2, seg.shape[1] // 2 + label = int(seg[cy, cx]) + if label == 0: + raise ValueError( + "central_seg_label: centre pixel is sky (label 0); the" + + " segmentation stamp is not centred on a detected object." + ) + return label + + +def seg_has_neighbour(seg, object_number): + """Whether a seg stamp holds any non-central, non-zero label. + + The per-object neighbour flag (shapepipe#776, a systematics-test hook): + ``True`` when the coadd segmentation stamp contains a footprint other than + the central object's, i.e. the object is blended. + + Parameters + ---------- + seg : numpy.ndarray + Segmentation stamp (integer SExtractor labels; 0 for sky). + object_number : int + Central object's segmentation label. + + Returns + ------- + bool + ``True`` if any non-zero label differs from ``object_number``. + """ + labels = seg[seg != 0] + return bool(labels.size and np.any(labels != object_number)) + + +def uberseg_weight(weight, seg, object_number, dilate_neighbour=0): """Zero a stamp's weight on neighbour-side pixels — the UberSeg treatment. UberSeg is Erin Sheldon's MEDS/ngmix neighbour mask (``esheldon/meds``, @@ -1046,7 +1195,17 @@ def uberseg_weight(weight, seg, object_number): Segmentation map on the same grid as ``weight``: 0 for sky, the SExtractor object number for each detected object's footprint. object_number : int - Segmentation label of the central object (its SExtractor ``NUMBER``). + Segmentation label of the central object (the centre-pixel label from + :func:`central_seg_label`, not necessarily the SExtractor ``NUMBER``). + dilate_neighbour : int, optional + Enlarge the neighbour mask by this many binary-dilation iterations + (4-connected, ~one pixel per iteration) on top of the base Voronoi + partition, to absorb the few-pixel coadd-vs-epoch registration offset + the coadd-seg overlay accepts (shapepipe#776, decision on seg source). + ``0`` (the default) recovers the pure Sheldon UberSeg mask, byte-for- + byte. The dilation is additive: it only ever zeros more pixels, never + restores a base-masked one, so over-masking a boundary pixel costs a + little central-object S/N but never leaks neighbour flux into the fit. Returns ------- @@ -1073,11 +1232,22 @@ def uberseg_weight(weight, seg, object_number): nearest_label = labels[nearest].reshape(seg.shape) weight[nearest_label != object_number] = 0.0 + # Enlarge the neighbour mask to absorb the coadd-vs-epoch offset: zero any + # pixel within ``dilate_neighbour`` of a neighbour footprint. Purely + # additive over the base Voronoi result above. + if dilate_neighbour > 0: + neighbour_mask = binary_dilation( + (seg != 0) & (seg != object_number), + iterations=dilate_neighbour, + ) + weight[neighbour_mask] = 0.0 + return weight def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, blend_handling="noisefill", seg=None, object_number=None, + dilate_neighbour=0, ): """bookkeeping for ngmix weights. runs on a single galaxy and epoch pixel scale and galaxy guess @@ -1107,6 +1277,9 @@ def prepare_ngmix_weights( object_number : int, optional Central object's segmentation label. Required for ``blend_handling="uberseg"``; ignored otherwise. + dilate_neighbour : int, optional + Neighbour-mask dilation iterations, passed to :func:`uberseg_weight` + under ``blend_handling="uberseg"``; ignored otherwise. Returns ------- @@ -1165,10 +1338,12 @@ def prepare_ngmix_weights( raise ValueError( "blend_handling='uberseg' requires a segmentation map and the" + " central object_number; none reached prepare_ngmix_weights." - + " The seg-map source is gated (see CosmoStat/shapepipe issue" - + " on UberSeg segmentation plumbing)." + + " Set SEG_VIGNET_PATH on the ngmix run (see" + + " CosmoStat/shapepipe#776)." ) - weight_map = uberseg_weight(weight_map, seg, object_number) + weight_map = uberseg_weight( + weight_map, seg, object_number, dilate_neighbour=dilate_neighbour + ) elif (~mask).any(): # noisefill (default): replace masked pixels with a noise realisation. gal_masked[~mask] = noise_img_gal[~mask] @@ -1179,6 +1354,7 @@ def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="hsm", wcs_full=None, ra=None, dec=None, blend_handling="noisefill", seg=None, object_number=None, + dilate_neighbour=0, ): """Build an ngmix Observation for a single galaxy epoch. @@ -1226,6 +1402,9 @@ def make_ngmix_observation( object_number : int, optional Central object's segmentation label. Required for ``blend_handling="uberseg"`` (ignored otherwise). + dilate_neighbour : int, optional + Neighbour-mask dilation iterations passed through to + :func:`prepare_ngmix_weights` under ``blend_handling="uberseg"``. Returns ------- @@ -1253,6 +1432,7 @@ def make_ngmix_observation( gal_masked, weight_map, noise_img = prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=bkg_rms, blend_handling=blend_handling, seg=seg, object_number=object_number, + dilate_neighbour=dilate_neighbour, ) if centroid_source == "hsm": @@ -1472,7 +1652,7 @@ def make_runners(prior, flux_guess, rng): def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="hsm", - blend_handling="noisefill", object_number=None, + blend_handling="noisefill", object_number=None, dilate_neighbour=0, ): """Do Ngmix Metacal. @@ -1501,8 +1681,13 @@ def do_ngmix_metacal( historical behaviour. ``"uberseg"`` consumes ``stamp.segs`` and ``object_number``. object_number : int, optional - Central object's segmentation label (its SExtractor ``NUMBER``). - Required for ``blend_handling="uberseg"`` (ignored otherwise). + Central object's segmentation label — the centre-pixel label from + :func:`central_seg_label` (not the SExtractor ``NUMBER``, which is no + longer plumbed through; shapepipe#776). Required for + ``blend_handling="uberseg"`` (ignored otherwise). + dilate_neighbour : int, optional + Neighbour-mask dilation iterations passed through to + :func:`make_ngmix_observation` under ``blend_handling="uberseg"``. Returns ------- @@ -1536,6 +1721,7 @@ def do_ngmix_metacal( blend_handling=blend_handling, seg=stamp.segs[n_e] if n_e < len(stamp.segs) else None, object_number=object_number, + dilate_neighbour=dilate_neighbour, ) gal_obs_list.append(gal_obs) From fa1b0dd280e5603368ecb8f332a58e7d3fb0e1ce Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Jul 2026 22:41:37 +0200 Subject: [PATCH 4/8] ngmix runner + configs: SEG_VIGNET_PATH, DILATE_NEIGHBOUR, seg vignetmaker run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the coadd-seg UberSeg inputs through the runner and example pipeline. - ngmix_runner reads SEG_VIGNET_PATH (optional, required for uberseg; missing file -> error) and DILATE_NEIGHBOUR (int, default 1), passing them to Ngmix. - config_tile_Sx.ini emits the coadd SEGMENTATION check image alongside BACKGROUND (CLI CHECKIMAGE overrides the .sex file, so no .sex edit). - config_tile_PiViVi_canfar_sx.ini adds VIGNETMAKER_RUNNER_RUN_3, a CLASSIC/PIX run cutting 51x51 integer seg stamps from segmentation{NUM}.fits at the tile catalogue centres — the same grid as the coadd VIGNET, so the overlay is well-defined. - config_tile_Ng_template.ini documents BLEND_HANDLING, SEG_VIGNET_PATH and DILATE_NEIGHBOUR (commented; uberseg requires the seg path). Co-Authored-By: Claude Fable 5 --- example/cfis/config_tile_Ng_template.ini | 20 ++++++++++ example/cfis/config_tile_PiViVi_canfar_sx.ini | 38 ++++++++++++++++++- example/cfis/config_tile_Sx.ini | 2 +- src/shapepipe/modules/ngmix_runner.py | 28 ++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/example/cfis/config_tile_Ng_template.ini b/example/cfis/config_tile_Ng_template.ini index 5bfbc2c79..ed6767a9e 100644 --- a/example/cfis/config_tile_Ng_template.ini +++ b/example/cfis/config_tile_Ng_template.ini @@ -83,5 +83,25 @@ PIXEL_SCALE = 0.186 # noisy for stars and flagged as incorrect — see #767). CENTROID_SOURCE = wcs +# BLEND_HANDLING (optional): neighbour treatment. "noisefill" (default, +# historical) replaces a neighbour's pixels with a noise realisation; +# "uberseg" hard-masks (weight -> 0) every pixel closer to a neighbour's +# segmentation footprint than to the central object. "uberseg" REQUIRES +# SEG_VIGNET_PATH below. +# BLEND_HANDLING = uberseg + +# SEG_VIGNET_PATH (optional): coadd-frame SExtractor segmentation vignets +# (the CLASSIC-mode VIGNETMAKER_RUNNER_RUN_3 seg output), row-aligned to the +# tile catalogue and on the same 51x51 grid as the coadd VIGNET. Required for +# BLEND_HANDLING = uberseg; when set, the file must exist for every tile +# (missing file -> error). Omit for the noise-fill path. +# SEG_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_3/output/seg_vignet{file_number_string}.fits + +# DILATE_NEIGHBOUR (optional): binary-dilation iterations enlarging the uberseg +# neighbour mask, to absorb the few-pixel coadd-vs-epoch seg-overlay offset. +# Ignored unless BLEND_HANDLING = uberseg. Default 1 (~one pixel); 0 recovers +# the pure Sheldon UberSeg mask. +# DILATE_NEIGHBOUR = 1 + ID_OBJ_MIN = X ID_OBJ_MAX = X diff --git a/example/cfis/config_tile_PiViVi_canfar_sx.ini b/example/cfis/config_tile_PiViVi_canfar_sx.ini index ca7efd9ec..cb72e1c11 100644 --- a/example/cfis/config_tile_PiViVi_canfar_sx.ini +++ b/example/cfis/config_tile_PiViVi_canfar_sx.ini @@ -21,7 +21,7 @@ RUN_NAME = run_sp_tile_PiViVi # Module name, single string or comma-separated list of valid module runner names #MODULE = psfex_interp_runner, -MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner +MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner, vignetmaker_runner # Parallel processing mode, SMP or MPI MODE = SMP @@ -166,3 +166,39 @@ PREFIX = ME_IMAGE_EXP_DIR = $SP_EXP ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner ME_IMAGE_PATTERN = flag, image, weight, background, background_rms + + +[VIGNETMAKER_RUNNER_RUN_3] + +# Cut per-object coadd-frame segmentation stamps from the tile SExtractor +# SEGMENTATION check image (config_tile_Sx.ini: CHECKIMAGE = BACKGROUND, +# SEGMENTATION). Integer labels, no interpolation, zero-padded — CLASSIC mode +# guarantees this. Row-aligned to the tile catalogue on the same XWIN/YWIN +# centres and 51x51 grid as the coadd VIGNET, so ngmix can overlay the seg +# stamp directly for uberseg neighbour masking (shapepipe#776). + +INPUT_DIR = run_sp_tile_Sx:sextractor_runner + +FILE_PATTERN = sexcat, segmentation + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# CLASSIC: cut stamps at object positions in the coadd (tile) frame. +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels — MUST equal the coadd VIGNET size (51) so the seg +# stamp overlays the galaxy stamp on an identical grid. +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = seg diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini index 085b4fb5a..0409c4155 100644 --- a/example/cfis/config_tile_Sx.ini +++ b/example/cfis/config_tile_Sx.ini @@ -97,7 +97,7 @@ BKG_FROM_HEADER = False # BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, # MINIBACK_RMS, -BACKGROUND, #FILTERED, # OBJECTS, -OBJECTS, SEGMENTATION, APERTURES -CHECKIMAGE = BACKGROUND +CHECKIMAGE = BACKGROUND, SEGMENTATION # File name suffix for the output sextractor files (optional) SUFFIX = sexcat diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 378a72a78..e8f9806e1 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -65,6 +65,24 @@ def ngmix_runner( else: input_file_list = input_file_list[:6] + # SEG_VIGNET_PATH (optional): coadd-frame SExtractor segmentation vignets + # (a CLASSIC-mode vignetmaker output), row-aligned to the tile catalogue. + # Required for BLEND_HANDLING = uberseg; when set, the file must exist for + # every tile (missing file -> error). Read on Tile_cat, not via Vignet, so + # it is threaded to Ngmix as its own argument rather than into + # input_file_list. + if config.has_option(module_config_sec, "SEG_VIGNET_PATH"): + seg_vignet_path = config.getexpanded( + module_config_sec, + "SEG_VIGNET_PATH", + ).format(file_number_string=file_number_string) + if not os.path.exists(seg_vignet_path): + raise FileNotFoundError( + f"Segmentation vignet file not found: {seg_vignet_path}" + ) + else: + seg_vignet_path = None + # Batch save option if config.has_option(module_config_sec, "SAVE_BATCH"): save_batch = config.getint( @@ -97,6 +115,14 @@ def ngmix_runner( else: blend_handling = "noisefill" + # DILATE_NEIGHBOUR (optional): binary-dilation iterations enlarging the + # uberseg neighbour mask, to absorb the few-pixel coadd-vs-epoch seg-overlay + # offset. Ignored unless BLEND_HANDLING = uberseg. Default 1 (~one pixel). + if config.has_option(module_config_sec, "DILATE_NEIGHBOUR"): + dilate_neighbour = config.getint(module_config_sec, "DILATE_NEIGHBOUR") + else: + dilate_neighbour = 1 + # Initialise class instance ngmix_inst = Ngmix( input_file_list, @@ -111,6 +137,8 @@ def ngmix_runner( id_obj_max=id_obj_max, centroid_source=centroid_source, blend_handling=blend_handling, + seg_cat_path=seg_vignet_path, + dilate_neighbour=dilate_neighbour, ) # Process ngmix shape measurement and metacalibration From fa6e0016f47cfc753fc767dff48daa9171a33f18 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Jul 2026 22:41:49 +0200 Subject: [PATCH 5/8] make_cat: NGMIX_NEIGHBOUR_FLAG column to the final catalogue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propagate the per-object blend flag (shapepipe#776) from the ngmix catalogue into the final catalogue as NGMIX_NEIGHBOUR_FLAG — a galaxy-only, OBJECT/ SHEAR-less metadata column alongside NGMIX_N_EPOCH and NGMIX_MCAL_TYPES_FAIL, enabling the systematics tests Axel requested. Added to final_cat.param so it survives the final selection. Co-Authored-By: Claude Fable 5 --- example/cfis/final_cat.param | 3 +++ .../modules/make_cat_package/make_cat.py | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/example/cfis/final_cat.param b/example/cfis/final_cat.param index 8ddfebd37..00bcb3f73 100644 --- a/example/cfis/final_cat.param +++ b/example/cfis/final_cat.param @@ -26,6 +26,9 @@ NGMIX_G2_PSF_ORIG_NOSHEAR N_EPOCH NGMIX_N_EPOCH +# Blend flag: coadd seg stamp held a non-central footprint (shapepipe#776) +NGMIX_NEIGHBOUR_FLAG + ## Shape measurement outputs ## Ngmix: model fitting diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 32855bdd6..24e72693a 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -327,9 +327,11 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): Save the NGMIX catalogue into the final one. Column grammar: ``NGMIX[m]_[_ERR][_]_``, - plus three OBJECT/SHEAR-less per-object metadata columns + plus four OBJECT/SHEAR-less per-object metadata columns (``NGMIX[m]_MCAL_FLAGS``, ``NGMIX_N_EPOCH``, - ``NGMIX_MCAL_TYPES_FAIL``). The galaxy is the implicit default object + ``NGMIX_MCAL_TYPES_FAIL``, ``NGMIX_NEIGHBOUR_FLAG`` — the last a blend + flag set when the coadd seg stamp held a non-central footprint, + shapepipe#776). The galaxy is the implicit default object and carries NO ``OBJECT`` token (``NGMIX_G1_NOSHEAR``, dropping the ``GAL`` segment carried by the pre-#761 names). The explicit PSF objects are ``PSF_ORIG`` @@ -369,6 +371,9 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): #return err_msg ngmix_mcal_types_fail = ngmix_cat_file.get_data()["mcal_types_fail"] + # Per-object blend flag (shapepipe#776): the seg stamp held a + # non-central footprint. Galaxy-only (non-moments), like N_EPOCH. + ngmix_neighbour_flag = ngmix_cat_file.get_data()["neighbour_flag"] # Needed in both moments and non-moments modes (used unconditionally # below), so read them outside the branch. ngmix_mcal_flags = ngmix_cat_file.get_data()["mcal_flags"] @@ -384,6 +389,7 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): self._add2dict("NGMIX_N_EPOCH", np.zeros(n_obj)) self._add2dict("NGMIX_MCAL_TYPES_FAIL", np.zeros(n_obj)) + self._add2dict("NGMIX_NEIGHBOUR_FLAG", np.zeros(n_obj)) prefix = f"NGMIX{m}" @@ -535,6 +541,11 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): ngmix_mcal_types_fail[ind[0]], idx, ) + self._add2dict( + "NGMIX_NEIGHBOUR_FLAG", + ngmix_neighbour_flag[ind[0]], + idx, + ) ngmix_cat_file.close() From 3bd759cdfcedb2ff3c429dff99db2514bcb37bb8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Jul 2026 22:48:44 +0200 Subject: [PATCH 6/8] tests: cover centre-pixel ID, neighbour flag, and uberseg dilation Extend test_ngmix_uberseg with: - central_seg_label: reads the centre-pixel label (not "label 1"), and fails fast on a sky centre. - seg_has_neighbour: true iff a non-central footprint is present. - uberseg dilation: dilate=0 is bit-identical to the pure Sheldon mask (and the O(N^2) brute force); dilate>0 zeros a strict superset once the grown neighbour footprint crosses the Voronoi bisector into the central cell. - noisefill ignores seg/dilate_neighbour kwargs (additive-safety). Add neighbour_flag to the compile_results / make_cat synthetic fixtures (test_ngmix, test_make_cat) so the new per-object column round-trips, and assert NGMIX_NEIGHBOUR_FLAG lands in the final catalogue. Co-Authored-By: Claude Fable 5 --- tests/module/test_make_cat.py | 13 +++- tests/module/test_ngmix.py | 1 + tests/module/test_ngmix_uberseg.py | 111 +++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 3 deletions(-) diff --git a/tests/module/test_make_cat.py b/tests/module/test_make_cat.py index d5d249429..3f08fadc7 100644 --- a/tests/module/test_make_cat.py +++ b/tests/module/test_make_cat.py @@ -6,8 +6,9 @@ ``GAL`` token (``NGMIX_G1_NOSHEAR``, never ``NGMIX_G1_GAL_NOSHEAR``), while the PSF families keep an explicit object token — ``NGMIX_[_ERR]__`` for ``PSF_ORIG``/``PSF_RECONV`` — -plus the three OBJECT/SHEAR-less metadata columns (``NGMIX[m]_MCAL_FLAGS``, -``NGMIX_N_EPOCH``, ``NGMIX_MCAL_TYPES_FAIL``). The original image PSF +plus the four OBJECT/SHEAR-less metadata columns (``NGMIX[m]_MCAL_FLAGS``, +``NGMIX_N_EPOCH``, ``NGMIX_MCAL_TYPES_FAIL``, ``NGMIX_NEIGHBOUR_FLAG``). The +original image PSF (``PSF_ORIG``) and the metacal reconvolution kernel (``PSF_RECONV``) are independent fits of *different* PSFs, no longer the single aliased value of the pre-#749 code. @@ -34,6 +35,7 @@ def info(self, *_args, **_kwargs): "id", "n_epoch_model", "mcal_types_fail", + "neighbour_flag", "nfev_fit", "g1", "g1_err", "g2", "g2_err", "T", "T_err", @@ -62,6 +64,7 @@ def _ngmix_row(obj_id): "id": obj_id, "n_epoch_model": 3, "mcal_types_fail": 0, + "neighbour_flag": 1, "nfev_fit": 7, "g1": 0.10, "g1_err": 0.011, "g2": -0.20, "g2_err": 0.022, "T": 0.30, "T_err": 0.033, @@ -149,7 +152,10 @@ def test_save_ngmix_data_uses_new_grammar_and_no_old_names(tmp_path): assert col in out, f"missing {col}" # Object-level metadata columns carry no OBJECT/SHEAR token. - for col in ("NGMIX_MCAL_FLAGS", "NGMIX_N_EPOCH", "NGMIX_MCAL_TYPES_FAIL"): + for col in ( + "NGMIX_MCAL_FLAGS", "NGMIX_N_EPOCH", "NGMIX_MCAL_TYPES_FAIL", + "NGMIX_NEIGHBOUR_FLAG", + ): assert col in out, f"missing {col}" # No old name survives anywhere in the produced columns. @@ -271,6 +277,7 @@ def test_save_ngmix_data_matches_module_serialised_catalogue(tmp_path): "obj_id": oid, "n_epoch_model": row["n_epoch_model"], "mcal_types_fail": row["mcal_types_fail"], + "neighbour_flag": row["neighbour_flag"], "mcal_flags": row["mcal_flags"], } for key in NGMIX_KEYS: diff --git a/tests/module/test_ngmix.py b/tests/module/test_ngmix.py index 139c18d4c..4951a42f1 100644 --- a/tests/module/test_ngmix.py +++ b/tests/module/test_ngmix.py @@ -175,6 +175,7 @@ def _fake_metacal_result(T, T_err, T_psf, T_psf_err): "obj_id": 1, "n_epoch_model": 1, "mcal_types_fail": 0, + "neighbour_flag": 0, # original image PSF (psfex/mccd) family "g1_psf_orig": ORIG_PSF_G[0], "g2_psf_orig": ORIG_PSF_G[1], diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 98a586d73..615a12fc9 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -20,7 +20,9 @@ from scipy import ndimage from shapepipe.modules.ngmix_package.ngmix import ( + central_seg_label, prepare_ngmix_weights, + seg_has_neighbour, uberseg_weight, ) @@ -101,6 +103,98 @@ def test_uberseg_no_neighbour_is_passthrough(): npt.assert_array_equal(uberseg_weight(weight, empty, object_number=1), weight) +# --- central_seg_label: centre-pixel identification (#776 decision 3) ------- + +def test_central_seg_label_is_the_centre_pixel(): + """The central label is read from the centre pixel, not assumed to be 1.""" + seg, centre, _ = two_object_seg() + assert central_seg_label(seg) == 1 + + # Slide the neighbour footprint onto the stamp centre: the centre-pixel + # label is now 2, proving the id comes from geometry, not "label 1". + npix = seg.shape[0] + c = npix // 2 + yy, xx = np.indices((npix, npix)) + shifted = np.zeros_like(seg) + shifted[(yy - c) ** 2 + (xx - c) ** 2 <= 3 ** 2] = 2 + assert central_seg_label(shifted) == 2 + + +def test_central_seg_label_sky_centre_raises(): + """A seg stamp whose centre pixel is sky (label 0) fails fast.""" + empty = np.zeros((21, 21), dtype=np.int32) + with pytest.raises(ValueError, match="centre pixel is sky"): + central_seg_label(empty) + + +# --- seg_has_neighbour: the per-object blend flag (#776 decision 4) --------- + +def test_seg_has_neighbour(): + """True iff a non-central, non-zero label is present.""" + seg, _, _ = two_object_seg() + assert seg_has_neighbour(seg, 1) is True + + # Only the central object present -> not blended. + solo = np.zeros((21, 21), dtype=np.int32) + solo[8:13, 8:13] = 1 + assert seg_has_neighbour(solo, 1) is False + + # Empty seg -> not blended. + assert seg_has_neighbour(np.zeros((21, 21), dtype=np.int32), 1) is False + + # A lone footprint that *is* the central object -> not blended, even + # though its label differs from a nominal "1". + assert seg_has_neighbour(solo * 7, 7) is False + + +# --- uberseg dilation: additive neighbour-mask enlargement (#776 dec. 2) ---- + +def test_uberseg_dilation_grows_neighbour_mask(): + """dilate_neighbour>0 zeros a strict superset of the base (dilate=0) mask, + and every base-masked pixel stays masked (additive-only). + + Geometric subtlety: for well-separated objects the base Voronoi cut (the + perpendicular bisector) already masks the whole neighbour-side half-plane, + so dilating the neighbour footprint bites only when its grown boundary + crosses the bisector into the central Voronoi cell. A few iterations + guarantee that crossing here (sep=8, r=3 -> ~2px footprint gap).""" + seg, _, _ = two_object_seg(npix=41, sep=8) + weight = np.ones_like(seg, dtype=float) + + out0 = uberseg_weight(weight, seg, object_number=1, dilate_neighbour=0) + out3 = uberseg_weight(weight, seg, object_number=1, dilate_neighbour=3) + + # dilate=0 reproduces the validated no-dilation result byte-for-byte. + npt.assert_array_equal( + out0, uberseg_weight(weight, seg, object_number=1) + ) + # Every pixel masked at dilate=0 is still masked at dilate=3 (additive). + assert np.all(out3[out0 == 0.0] == 0.0) + # And strictly more pixels are masked once the dilation crosses the + # bisector into the central cell. + assert (out3 == 0.0).sum() > (out0 == 0.0).sum() + + +def test_uberseg_dilation_zero_is_pure_sheldon(): + """dilate_neighbour=0 is bit-identical to the default (no-kwarg) call and + to the O(N^2) brute-force nearest-segment rule.""" + seg, _, _ = two_object_seg(npix=25, sep=8) + weight = np.ones_like(seg, dtype=float) + + out = uberseg_weight(weight, seg, object_number=1, dilate_neighbour=0) + npt.assert_array_equal(out, uberseg_weight(weight, seg, object_number=1)) + + obj = np.argwhere(seg != 0) + labels = seg[seg != 0] + brute = np.ones_like(weight) + for i in range(seg.shape[0]): + for j in range(seg.shape[1]): + d2 = (i - obj[:, 0]) ** 2 + (j - obj[:, 1]) ** 2 + if labels[np.argmin(d2)] != 1: + brute[i, j] = 0.0 + npt.assert_array_equal(out, brute) + + def test_uberseg_matches_bruteforce_nearest_segment(): """The cKDTree result equals the O(N^2) brute-force nearest-segment rule (Sheldon's non-C fallback) it stands in for.""" @@ -159,6 +253,23 @@ def test_noisefill_default_is_byte_identical_to_legacy(): npt.assert_array_equal(noise_out, noise_exp) +def test_noisefill_ignores_seg_and_dilate_kwargs(): + """Under noisefill, passing seg / dilate_neighbour changes nothing: the + result matches the plain default call on the same RNG stream.""" + gal, flag, weight = _gal_flag_weight() + seg, _, _ = two_object_seg(npix=gal.shape[0], sep=12) + + base = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(321), + ) + with_kwargs = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(321), + seg=seg, object_number=1, dilate_neighbour=3, + ) + for a, b in zip(with_kwargs, base): + npt.assert_array_equal(a, b) + + def test_uberseg_hard_masks_weight_and_leaves_image_untouched(): """uberseg: image returned untouched, weight zeroed on neighbour-side and flagged pixels, positive on the central core.""" From b8a6d406771cd7b64ece875057dbcc2ac73e0141 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Jul 2026 23:37:02 +0200 Subject: [PATCH 7/8] tests: add neighbour_flag column to the PSF property-test fixtures The two Hypothesis property files that drive make_cat's SaveCatalogue._save_ngmix_data still built synthetic ngmix catalogues on the pre-#776 column set. Since the coadd-seg UberSeg work, _save_ngmix_data reads "neighbour_flag" unconditionally and emits NGMIX_NEIGHBOUR_FLAG, so those fixtures were one column short of the real writer contract: - test_psf_averaging_properties.py: the writer's get_data()["neighbour_flag"] raised on the column-less cat, failing test_absent_obj_id_keeps_sentinel_fill. - test_psf_grammar_properties.py: same missing-column read, and once fed the column the emitted NGMIX_NEIGHBOUR_FLAG was off-grammar for GRAMMAR_RE / FROZEN_GRAMMAR_RE. On a failing property Hypothesis runs its full shrink search, which is what made the sweep look hung at ~21% for minutes rather than fail promptly. Fix, matching the update commit 3bd759cd already made to test_ngmix / test_make_cat: add "neighbour_flag" to each file's NGMIX key + INT key lists and base/measured row, add NGMIX_NEIGHBOUR_FLAG to the averaging test's _SENTINELS (so its zero-fill is now covered) and to the grammar regexes + frozen-grammar accept list (so the new metadata column is a recognised token). Additive only; no assertion weakened. Full tests/module tests/unit sweep: 332 passed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JxsRFJfbvpnx7mgAtqfJqe --- tests/module/test_psf_averaging_properties.py | 6 ++++-- tests/module/test_psf_grammar_properties.py | 20 +++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/module/test_psf_averaging_properties.py b/tests/module/test_psf_averaging_properties.py index 3afaa7bbe..a6ce2b782 100644 --- a/tests/module/test_psf_averaging_properties.py +++ b/tests/module/test_psf_averaging_properties.py @@ -250,12 +250,13 @@ def test_all_epochs_failed_raises_zero_division(flagged_specs): "NGMIX_N_EPOCH": 0.0, "NGMIX_MCAL_FLAGS": 0.0, "NGMIX_MCAL_TYPES_FAIL": 0.0, + "NGMIX_NEIGHBOUR_FLAG": 0.0, } # Per-key write format and a measured value distinct from every sentinel, so a # matched row is unmistakably "overwritten" and an absent row unmistakably not. _NGMIX_KEYS = [ - "id", "n_epoch_model", "mcal_types_fail", "nfev_fit", + "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", "nfev_fit", "g1", "g1_err", "g2", "g2_err", "T", "T_err", "flux", "flux_err", "s2n", "mag", "mag_err", "flags", "mcal_flags", "g1_psf_orig", "g2_psf_orig", "g1_err_psf_orig", "g2_err_psf_orig", @@ -264,7 +265,8 @@ def test_all_epochs_failed_raises_zero_division(flagged_specs): "T_psf_reconv", "T_err_psf_reconv", ] _INT_KEYS = { - "id", "n_epoch_model", "mcal_types_fail", "nfev_fit", "flags", "mcal_flags" + "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", "nfev_fit", + "flags", "mcal_flags" } _SHEAR_EXTS = ["1M", "1P", "2M", "2P", "NOSHEAR"] diff --git a/tests/module/test_psf_grammar_properties.py b/tests/module/test_psf_grammar_properties.py index dec5a758b..b0ac1ee47 100644 --- a/tests/module/test_psf_grammar_properties.py +++ b/tests/module/test_psf_grammar_properties.py @@ -4,9 +4,10 @@ hypothesis to pin three invariants of the renamed grammar ``NGMIX[m]_[_ERR][_]_`` (shapepipe#749, #761): the galaxy is the implicit default object and carries NO ``OBJECT`` token -(``NGMIX_G1_NOSHEAR``, never ``NGMIX_G1_GAL_NOSHEAR``), plus the three +(``NGMIX_G1_NOSHEAR``, never ``NGMIX_G1_GAL_NOSHEAR``), plus the four OBJECT/SHEAR-less metadata columns (``NGMIX[m]_MCAL_FLAGS``, ``NGMIX_N_EPOCH``, -``NGMIX_MCAL_TYPES_FAIL``), where the ORIGINAL image PSF (``PSF_ORIG``) and the +``NGMIX_MCAL_TYPES_FAIL``, ``NGMIX_NEIGHBOUR_FLAG``), where the ORIGINAL image +PSF (``PSF_ORIG``) and the metacal RECONVOLUTION kernel (``PSF_RECONV``) are independent fits of *different* PSFs: @@ -62,10 +63,11 @@ def info(self, *_args, **_kwargs): # shear-type extension. Integer-typed keys carry FITS format "K", the rest # "D"; everything else mirrors test_make_cat._ngmix_row. INT_KEYS = { - "id", "n_epoch_model", "mcal_types_fail", "nfev_fit", "flags", "mcal_flags", + "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", "nfev_fit", + "flags", "mcal_flags", } NGMIX_KEYS = [ - "id", "n_epoch_model", "mcal_types_fail", "nfev_fit", + "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", "nfev_fit", "g1", "g1_err", "g2", "g2_err", "T", "T_err", "flux", "flux_err", "s2n", "mag", "mag_err", @@ -106,7 +108,8 @@ def _base_row(obj_id): """One object's per-key values; PSF keys overwritten by the caller.""" return { "id": obj_id, - "n_epoch_model": 3, "mcal_types_fail": 0, "nfev_fit": 7, + "n_epoch_model": 3, "mcal_types_fail": 0, "neighbour_flag": 1, + "nfev_fit": 7, "g1": 0.10, "g1_err": 0.011, "g2": -0.20, "g2_err": 0.022, "T": 0.30, "T_err": 0.033, "flux": 100.0, "flux_err": 1.0, "s2n": 55.0, @@ -175,7 +178,7 @@ def _run_save_ngmix(ngmix_path, obj_ids, cat_size_target=None): _SHEAR = "NOSHEAR|1P|1M|2P|2M" GRAMMAR_RE = re.compile( rf"^NGMIXm?_(?:{_COMPONENT})(?:_ERR)?(?:_(?:{_OBJECT}))?_(?:{_SHEAR})$" - rf"|^NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH)$" + rf"|^NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH|NEIGHBOUR_FLAG)$" ) # The shipped final-catalogue param files, two levels up from tests/module/. @@ -323,7 +326,7 @@ def test_emitted_column_names_match_grammar(obj_ids, tmp_path_factory): """Every column _save_ngmix_data emits matches the grammar regex. The grammar is ``NGMIX[m]_[_ERR]__`` plus the - three metadata columns. The regex is anchored, so a legacy token (``_PSFo``, + four metadata columns. The regex is anchored, so a legacy token (``_PSFo``, ``_Tpsf``, ``NGMIX_ELL_*``) or a malformed name would fail to match. Run over hypothesis-varied object sets to confirm the emitted name set is input-independent and always well-formed. @@ -411,7 +414,7 @@ def test_param_file_ngmix_tokens_are_producible(param_path, obj_ids): # ngmix: galaxy (no object token) or explicit PSF_ORIG/PSF_RECONV. r"NGMIXm?_(?:G1|G2|T|SNR|FLUX|MAG|FLAGS)(?:_ERR)?" r"(?:_(?:PSF_ORIG|PSF_RECONV))?_(?:NOSHEAR|1P|1M|2P|2M)" - r"|NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH)" + r"|NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH|NEIGHBOUR_FLAG)" # HSM: g-type, explicit PSF/STAR object, singular FLAG; the multi-epoch # sink in make_cat._save_psf_data appends a bare epoch index. r"|HSM_(?:G1|G2|T)_(?:PSF|STAR)(?:_\d+)?" @@ -428,6 +431,7 @@ def test_param_file_ngmix_tokens_are_producible(param_path, obj_ids): "NGMIX_FLAGS_2P", "NGMIXm_G1_PSF_RECONV_NOSHEAR", "NGMIX_MCAL_FLAGS", + "NGMIX_NEIGHBOUR_FLAG", # per-object blend flag (shapepipe#776) "HSM_G1_PSF", "HSM_T_STAR", "HSM_FLAG_PSF", From af5ebb56e35f2096e9b82f2df10ddf8893075c37 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 12 Jul 2026 23:58:30 +0200 Subject: [PATCH 8/8] ngmix uberseg: obj_id is the authoritative central label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The central segmentation label is now the object's SExtractor NUMBER (obj_id) everywhere — uberseg mask and neighbour flag alike. Seg labels are NUMBERs from the same SE run, so obj_id names the central footprint directly and cannot be stolen by a neighbour when a few-pixel coadd-vs- epoch overlay offset lands a neighbour's label on the centre pixel (which would invert the mask). This also unifies the neighbour flag's semantics across blend_handling modes. central_seg_label is demoted to a diagnostic: Ngmix._check_central_seg_label cross-checks the centre pixel against obj_id, warns on a mismatch (a registration signal) but proceeds with obj_id, and raises if the seg stamp contains obj_id nowhere (object dropped by the per-object try/except, loud in the log). Tile_cat.get_data now fails fast if the seg VIGNET catalogue is not row-aligned to the tile catalogue (row-count mismatch, or a NUMBER column that disagrees) — a silent misalignment would hand every object the wrong footprint. Regression tests: uberseg __init__ without SEG_VIGNET_PATH, the runner's FileNotFoundError on a missing seg file, and the three _check_central_seg_label outcomes. Co-Authored-By: Claude Fable 5 --- src/shapepipe/modules/ngmix_package/ngmix.py | 114 +++++++++++++++---- tests/module/test_ngmix_uberseg.py | 112 ++++++++++++++++++ 2 files changed, 202 insertions(+), 24 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 939243026..993f47c19 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -181,7 +181,26 @@ def get_data(self, cat_path): SEx_catalogue=True, ) seg_cat.open() - self.seg = np.copy(seg_cat.get_data()['VIGNET']) + seg_data = seg_cat.get_data() + # The seg VIGNETs are indexed by tile-catalogue row (self.seg[i]), + # so the two catalogues MUST be row-aligned. Fail loud at load if + # they are not — a silent length/order mismatch would hand every + # object the wrong footprint and quietly corrupt every mask. + if len(seg_data) != len(self.obj_id): + raise ValueError( + f"SEG_VIGNET_PATH '{self.seg_cat_path}' has" + + f" {len(seg_data)} rows but the tile catalogue has" + + f" {len(self.obj_id)}; the segmentation vignets must be" + + " row-aligned to the tile catalogue." + ) + if 'NUMBER' in seg_data.dtype.names: + if not np.array_equal(seg_data['NUMBER'], self.obj_id): + raise ValueError( + f"SEG_VIGNET_PATH '{self.seg_cat_path}' NUMBER column" + + " does not match the tile catalogue NUMBER; the" + + " segmentation vignets are misaligned or reordered." + ) + self.seg = np.copy(seg_data['VIGNET']) seg_cat.close() class Postage_stamp(): @@ -706,6 +725,46 @@ def check_key(self, expccd_name_tmp, vign_cat, vignet_path): + f" file '{vignet_path}'" ) + def _check_central_seg_label(self, seg, obj_id): + """Cross-check the seg stamp's centre pixel against the object's NUMBER. + + The authoritative central label is ``obj_id`` (the SExtractor NUMBER); + this is a diagnostic on the coadd-vs-epoch overlay registration, not a + gate. Two outcomes: + + * The seg stamp does not contain ``obj_id`` anywhere — the object's own + footprint fell outside the cutout (bad centroid / dropped detection), + so uberseg has no central object to keep. Raise; the per-object + ``try/except`` in :meth:`process` drops it, loud in the log. + * The centre-pixel label is nonzero and ``!= obj_id`` — a few-pixel + registration offset put a neighbour on the centre pixel. Harmless for + the mask (which keys off ``obj_id``), but a signal worth surfacing: + log a warning and proceed. + + Parameters + ---------- + seg : numpy.ndarray + Coadd segmentation stamp for this object (integer labels). + obj_id : int + The object's SExtractor NUMBER — the authoritative central label. + """ + if not np.any(seg == obj_id): + raise ValueError( + f"seg stamp for object NUMBER={obj_id} contains that label" + + " nowhere; the object's footprint fell outside the cutout" + + " (bad centroid / registration). Cannot define the central" + + " object for uberseg." + ) + cy, cx = seg.shape[0] // 2, seg.shape[1] // 2 + centre_label = int(seg[cy, cx]) + if centre_label != 0 and centre_label != obj_id: + self._w_log.info( + f"uberseg: object NUMBER={obj_id} but seg centre pixel carries" + + f" label {centre_label}; a coadd-vs-epoch registration offset" + + " is placing a neighbour on the centre. Proceeding with the" + + " NUMBER (authoritative)." + ) + def process(self): """Process. @@ -767,14 +826,17 @@ def process(self): if tile_cat.flux is not None else 1.0 ) - # Central object's seg label from the centre pixel (uberseg), - # else the SExtractor NUMBER (noisefill ignores it). Fails loud - # on a mis-centred seg stamp — the object is then dropped below. - central_label = ( - central_seg_label(tile_cat.seg[i_tile]) - if self._blend_handling == "uberseg" - else obj_id - ) + # The central object's segmentation label IS its SExtractor + # NUMBER (obj_id): seg labels are the NUMBERs of the same SE run + # that produced the tile catalogue, so obj_id is authoritative + # and is used for both the uberseg mask and the neighbour flag. + # The centre pixel is only a diagnostic: a few-pixel coadd-vs- + # epoch overlay offset can put a NEIGHBOUR's label there, which + # would invert the mask — so we never trust it to define the + # central object. Cross-check it and warn on a mismatch (a + # registration signal), but proceed with obj_id (shapepipe#776). + if self._blend_handling == "uberseg": + self._check_central_seg_label(tile_cat.seg[i_tile], obj_id) res, psf_res, psf_orig_res = do_ngmix_metacal( stamp, prior, @@ -782,7 +844,7 @@ def process(self): self._rng, centroid_source=self._centroid_source, blend_handling=self._blend_handling, - object_number=central_label, + object_number=obj_id, dilate_neighbour=self._dilate_neighbour, ) except Exception as ee: @@ -797,7 +859,7 @@ def process(self): # non-zero label? Systematics-test hook (shapepipe#776); computed # from the raw coadd seg, 0 when uberseg is off / seg absent. res['neighbour_flag'] = int( - seg_has_neighbour(tile_cat.seg[i_tile], central_label) + seg_has_neighbour(tile_cat.seg[i_tile], obj_id) if getattr(tile_cat, "seg", None) is not None else 0 ) @@ -1096,14 +1158,16 @@ def get_noise(gal, weight, guess, pixel_scale, thresh=1.2): return sig_noise def central_seg_label(seg): - """Central object's label — the value at the seg stamp's centre pixel. + """Centre-pixel label of a seg stamp — a *diagnostic*, not the central id. - The central object is identified geometrically, from the centre pixel of - the coadd segmentation stamp, rather than by plumbing the SExtractor - ``NUMBER`` through the pipeline (shapepipe#776). The stamp is cut centred - on the object, so its central pixel lies in that object's footprint; this - keeps the seg stamp self-describing and robust to a few-pixel coadd-vs- - epoch registration offset that could otherwise mismatch a plumbed NUMBER. + The authoritative central label is the object's SExtractor ``NUMBER`` + (``obj_id``), which is plumbed straight through: segmentation labels ARE + the NUMBERs of the same SE run, so ``obj_id`` names the central footprint + directly (shapepipe#776). This helper reads the centre pixel only as a + registration cross-check — a few-pixel coadd-vs-epoch overlay offset can + put a NEIGHBOUR's label on the centre pixel, which is exactly why the + centre pixel must NOT define the central object (trusting it there would + invert the uberseg mask). See :meth:`Ngmix._check_central_seg_label`. Parameters ---------- @@ -1195,8 +1259,10 @@ def uberseg_weight(weight, seg, object_number, dilate_neighbour=0): Segmentation map on the same grid as ``weight``: 0 for sky, the SExtractor object number for each detected object's footprint. object_number : int - Segmentation label of the central object (the centre-pixel label from - :func:`central_seg_label`, not necessarily the SExtractor ``NUMBER``). + Segmentation label of the central object — its SExtractor ``NUMBER`` + (``obj_id``), authoritative because seg labels are the NUMBERs of the + same SE run. Not the centre-pixel label, which a few-pixel coadd-vs- + epoch offset can steal for a neighbour and so invert this mask. dilate_neighbour : int, optional Enlarge the neighbour mask by this many binary-dilation iterations (4-connected, ~one pixel per iteration) on top of the base Voronoi @@ -1681,10 +1747,10 @@ def do_ngmix_metacal( historical behaviour. ``"uberseg"`` consumes ``stamp.segs`` and ``object_number``. object_number : int, optional - Central object's segmentation label — the centre-pixel label from - :func:`central_seg_label` (not the SExtractor ``NUMBER``, which is no - longer plumbed through; shapepipe#776). Required for - ``blend_handling="uberseg"`` (ignored otherwise). + Central object's segmentation label — its SExtractor ``NUMBER`` + (``obj_id``), authoritative because seg labels are the NUMBERs of the + same SE run (shapepipe#776). Required for ``blend_handling="uberseg"`` + (ignored otherwise). dilate_neighbour : int, optional Neighbour-mask dilation iterations passed through to :func:`make_ngmix_observation` under ``blend_handling="uberseg"``. diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 615a12fc9..4ab4f43e2 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -18,8 +18,10 @@ import numpy.testing as npt import pytest from scipy import ndimage +from sqlitedict import SqliteDict from shapepipe.modules.ngmix_package.ngmix import ( + Ngmix, central_seg_label, prepare_ngmix_weights, seg_has_neighbour, @@ -27,6 +29,16 @@ ) +class _RecordingLogger: + """Minimal logger that records the messages passed to ``info``.""" + + def __init__(self): + self.messages = [] + + def info(self, msg, *_args, **_kwargs): + self.messages.append(msg) + + def two_object_seg(npix=41, sep=12, r_central=3, r_neighbour=3): """Stamp seg map: central object (label 1) at centre, neighbour (label 2) offset ``sep`` pixels along the column axis. Returns (seg, centre, neigh). @@ -307,3 +319,103 @@ def test_unknown_blend_handling_raises(): gal, weight, flag, np.random.RandomState(0), blend_handling="bogus", ) + + +# --- _check_central_seg_label: obj_id authoritative, centre pixel diagnostic - + +def _make_ngmix(tmp_path): + """Minimal Ngmix instance (empty sqlite vignets) with a recording logger.""" + names = ("gal", "bkg", "psf", "weight", "flag", "headers") + paths = [tmp_path / f"{name}.sqlite" for name in names] + for path in paths: + SqliteDict(str(path)).close() + log = _RecordingLogger() + ngmix = Ngmix( + ["tile_cat.fits"] + [str(p) for p in paths[:5]], + str(tmp_path), "-001-001", 30.0, 0.186, str(paths[5]), log, + ) + return ngmix, log + + +def test_check_central_seg_label_matching_number_is_silent(tmp_path): + """obj_id present and on the centre pixel: no warning, no raise.""" + ngmix, log = _make_ngmix(tmp_path) + seg, _, _ = two_object_seg() # central footprint has label 1 + log.messages.clear() # drop the __init__ seed log + ngmix._check_central_seg_label(seg, obj_id=1) + assert log.messages == [] + + +def test_check_central_seg_label_offset_centre_warns_but_proceeds(tmp_path): + """A neighbour's label on the centre pixel (registration offset) warns but + does not raise, as long as obj_id's footprint is present somewhere.""" + ngmix, log = _make_ngmix(tmp_path) + seg, centre, _ = two_object_seg() + # Overwrite the centre pixel with the neighbour label 2; obj_id=1 still + # has its footprint elsewhere on the stamp. + seg[centre] = 2 + ngmix._check_central_seg_label(seg, obj_id=1) + assert any("registration offset" in m for m in log.messages) + + +def test_check_central_seg_label_missing_number_raises(tmp_path): + """obj_id's footprint absent from the stamp entirely: fail fast.""" + ngmix, _ = _make_ngmix(tmp_path) + seg, _, _ = two_object_seg() # labels {0, 1, 2}; 99 is nowhere + with pytest.raises(ValueError, match="contains that label"): + ngmix._check_central_seg_label(seg, obj_id=99) + + +# --- FIX 3a: uberseg without seg_cat_path fails at construction ------------- + +def test_ngmix_init_uberseg_without_seg_cat_raises(tmp_path): + """blend_handling='uberseg' with seg_cat_path=None raises at __init__.""" + names = ("gal", "bkg", "psf", "weight", "flag", "headers") + paths = [tmp_path / f"{name}.sqlite" for name in names] + for path in paths: + SqliteDict(str(path)).close() + with pytest.raises(ValueError, match="requires SEG_VIGNET_PATH"): + Ngmix( + ["tile_cat.fits"] + [str(p) for p in paths[:5]], + str(tmp_path), "-001-001", 30.0, 0.186, str(paths[5]), + _RecordingLogger(), + blend_handling="uberseg", + seg_cat_path=None, + ) + + +# --- FIX 3b: runner raises when SEG_VIGNET_PATH is set but missing ---------- + +class _FakeConfig: + """Config stub: SEG_VIGNET_PATH is the only present option, and it resolves + to ``seg_path`` (a path the test leaves nonexistent).""" + + def __init__(self, seg_path): + self._seg_path = seg_path + + def getfloat(self, _sec, _key): + return 30.0 if _key == "MAG_ZP" else 0.186 + + def has_option(self, _sec, key): + return key == "SEG_VIGNET_PATH" + + def getexpanded(self, _sec, _key): + return self._seg_path + + +def test_runner_missing_seg_vignet_file_raises(tmp_path): + """SEG_VIGNET_PATH configured but the resolved file is absent -> the runner + fails fast with FileNotFoundError, before constructing Ngmix.""" + from shapepipe.modules.ngmix_runner import ngmix_runner + + input_file_list = ["tile_cat.fits"] + [f"in{i}.sqlite" for i in range(6)] + seg_path = str(tmp_path / "does_not_exist.fits") + with pytest.raises(FileNotFoundError, match="Segmentation vignet file"): + ngmix_runner( + input_file_list, + {"output": str(tmp_path)}, + "-001-001", + _FakeConfig(seg_path), + "NGMIX_RUNNER", + _RecordingLogger(), + )