Consume external healsparse masks: rasterize to pipeline flag images + per-band catalogue flags

example/cfis/config_exp_MaExt.ini

@@ -0,0 +1,52 @@
+## ShapePipe configuration file for exposure-CCD external-mask (healsparse)
+## rasterization. Reprojects the unified UNIONS healsparse mask through each
+## single-exposure single-CCD WCS and sums in the instrument flag file,
+## producing the pipeline_flag<num>.fits artifact consumed downstream.
+
+[DEFAULT]
+VERBOSE = True
+RUN_NAME = run_sp_exp_MaExt
+RUN_DATETIME = False
+
+[EXECUTION]
+MODULE = mask_ext_runner
+MODE = SMP
+
+[FILE]
+LOG_NAME = log_sp_exp
+RUN_LOG_NAME = log_run_sp
+INPUT_DIR = $SP_RUN/output
+OUTPUT_DIR = $SP_RUN/output
+
+[JOB]
+SMP_BATCH_SIZE = 1
+TIMEOUT = 96:00:00
+
+[MASK_EXT_RUNNER]
+
+# Parent module: single-exposure single-CCD images and instrument flags
+INPUT_DIR = last:split_exp_runner
+
+# Update numbering convention, accounting for HDU number of
+# single-exposure single-HDU files
+NUMBERING_SCHEME = -0000000-0
+
+# Path of the external healsparse mask file (band-agnostic; r-band for shear)
+MASK_PATH = $SP_CONFIG/mask_r.hsp
+
+# Healsparse bit value -> output flag value mapping.
+# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks
+# (True = masked, e.g. mask_r_nside131072.hsp) use 1:<flag>.
+BIT_FLAG_MAP = 64:1
+
+# Flag value for pixels outside the healsparse footprint (0 = unflagged)
+OFF_MAP_FLAG = 0
+
+# External instrument flag file: summed into the rasterized mask
+USE_EXT_FLAG = True
+
+# HDU of the external instrument flag FITS file (optional, default 0)
+HDU = 0
+
+# File name prefix for the output flag files
+PREFIX = pipeline

example/cfis/config_tile_MaExt.ini

@@ -0,0 +1,53 @@
+## ShapePipe configuration file for tile external-mask (healsparse) rasterization
+## Rasterizes the unified UNIONS healsparse mask onto each tile pixel grid,
+## producing the pipeline_flag<num>.fits artifact consumed downstream.
+
+[DEFAULT]
+VERBOSE = True
+RUN_NAME = run_sp_tile_MaExt
+RUN_DATETIME = False
+
+[EXECUTION]
+MODULE = mask_ext_runner
+MODE = SMP
+
+[FILE]
+LOG_NAME = log_sp_exp
+RUN_LOG_NAME = log_run_sp
+INPUT_DIR = $SP_RUN/output
+OUTPUT_DIR = $SP_RUN/output
+
+[JOB]
+SMP_BATCH_SIZE = 1
+TIMEOUT = 96:00:00
+
+[MASK_EXT_RUNNER]
+
+# Input directory, tile image
+INPUT_DIR = run_sp_tile_Git:get_images_runner, last:uncompress_fits_runner
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+# Input file pattern(s): only the tile image is needed (WCS + pixel grid)
+FILE_PATTERN = CFIS_image
+
+# FILE_EXT (optional) list of string extensions to identify input files
+FILE_EXT = .fits
+
+# Path of the external healsparse mask file (band-agnostic; r-band for shear)
+MASK_PATH = $SP_CONFIG/mask_r.hsp
+
+# Healsparse bit value -> output flag value mapping.
+# Integer bit-flag masks use per-band bits (e.g. 64 = r); boolean masks
+# (True = masked, e.g. mask_r_nside131072.hsp) use 1:<flag>.
+BIT_FLAG_MAP = 64:1
+
+# Flag value for pixels outside the healsparse footprint (0 = unflagged)
+OFF_MAP_FLAG = 0
+
+# No external instrument flag for tiles
+USE_EXT_FLAG = False
+
+# File name prefix for the output flag files
+PREFIX = pipeline

pyproject.toml

@@ -24,6 +24,8 @@ dependencies = [
     "cs_util>=0.2.1",
     "galsim>=2.8",
     "h5py",
+    "healsparse",
+    "hpgeom",
     "joblib>=1.4",
     "matplotlib>=3.10",
     "mccd>=1.2.4",

src/shapepipe/modules/make_cat_package/make_cat.py

@@ -209,6 +209,68 @@ def save_sm_data(
     return n_obj
 
 
+def parse_mask_ext_paths(paths_str):
+    """Parse Mask Ext Paths.
+
+    Parse the ``MASK_EXT_PATHS`` config value into a ``band -> path`` mapping.
+
+    Parameters
+    ----------
+    paths_str : str
+        Comma-separated ``band:path`` pairs, e.g.
+        ``u:/path/mask_u.hsp, g:/path/mask_g.hsp``
+
+    Returns
+    -------
+    dict
+        Mapping from band name to healsparse map path
+
+    """
+    band_paths = {}
+    for pair in paths_str.split(","):
+        band, path = pair.split(":", 1)
+        band_paths[band.strip()] = path.strip()
+
+    return band_paths
+
+
+def save_mask_ext_data(final_cat_file, band_paths, w_log):
+    """Save External Mask Data.
+
+    Query per-band external healsparse masks at each object's world position
+    and write one ``MASK_<BAND>`` column per band into the final catalogue.
+    Object positions are read from the SExtractor windowed world coordinates
+    (``XWIN_WORLD`` = RA, ``YWIN_WORLD`` = Dec, both in degrees) carried in the
+    ``RESULTS`` extension. Objects falling outside a map's coverage receive
+    that map's sentinel value (``healsparse.HealSparseMap.get_values_pos``
+    returns the map's sentinel — ``-1`` for integer maps — verbatim), which is
+    the documented off-map flag.
+
+    Parameters
+    ----------
+    final_cat_file : file_io.FITSCatalogue
+        Final catalogue
+    band_paths : dict
+        Mapping from band name to healsparse map path
+    w_log : logging.Logger
+        Logging instance
+
+    """
+    import healsparse
+
+    final_cat_file.open()
+    ra = np.copy(final_cat_file.get_data()["XWIN_WORLD"])
+    dec = np.copy(final_cat_file.get_data()["YWIN_WORLD"])
+
+    for band, path in band_paths.items():
+        w_log.info(f"Query external mask for band {band}: {path}")
+        mask_map = healsparse.HealSparseMap.read(path)
+        values = mask_map.get_values_pos(ra, dec, lonlat=True)
+        final_cat_file.add_col(f"MASK_{band}", np.asarray(values))
+
+    final_cat_file.close()
+
+
 class SaveCatalogue:
     """Save Catalogue.
 

src/shapepipe/modules/make_cat_runner.py

@@ -137,4 +137,14 @@ def make_cat_runner(
     if save_psf:
         err_msg = sc_inst.process("psf", galaxy_psf_path)
 
+    # Optional per-band external healsparse mask lookup (UNIONS-WL/spherex#38):
+    # add one MASK_<BAND> column per band, queried at each object's world
+    # position. Absent config is a strict no-op.
+    if config.has_option(module_config_sec, "MASK_EXT_PATHS"):
+        band_paths = make_cat.parse_mask_ext_paths(
+            config.getexpanded(module_config_sec, "MASK_EXT_PATHS")
+        )
+        w_log.info("Save external mask data")
+        make_cat.save_mask_ext_data(final_cat_file, band_paths, w_log)
+
     return None, None

src/shapepipe/modules/mask_ext_package/__init__.py

@@ -0,0 +1,71 @@
+"""MASK EXT MODULE.
+
+This package contains the module for ``mask_ext``.
+
+:Author: Cail Daley
+
+:Parent module: ``split_exp_runner`` (exposures) or ``get_images_runner`` /
+                ``uncompress_fits_runner`` (tiles), or None
+
+:Input: Single image (tile or single-exposure single-CCD) and, optionally, an
+        external instrument flag file
+
+:Output: Per-image pixel flag file
+
+Description
+===========
+
+This module produces the ShapePipe per-image pixel flag file
+``<PREFIX>_flag<num>.fits`` by rasterizing an **external healsparse mask** onto
+the image pixel grid, rather than generating masks internally (the job of the
+``mask`` module). It is the ShapePipe consumer of the unified UNIONS healsparse
+mask products (PhotoPipe footprint + bright stars, MaxiMask, manual galaxy
+masks), merged and stored as ``healsparse.HealSparseMap`` files.
+
+For each image the module:
+
+1. Reads the image header into an ``astropy.wcs.WCS``.
+2. Evaluates the pixel grid to (RA, Dec) in row chunks (bounded memory).
+3. Queries ``HealSparseMap.get_values_pos`` at every pixel centre.
+4. Maps healsparse bit values to ShapePipe flag values through the
+   config-driven ``BIT_FLAG_MAP``, producing an ``int16`` flag image.
+5. Optionally sums in an external instrument flag image (``USE_EXT_FLAG``),
+   matching the combination semantics of the ``mask`` module.
+6. Writes ``<PREFIX>_flag<num>.fits`` carrying the image WCS in its header.
+
+Everything downstream (SExtractor ``IMAFLAGS_ISO``, setools star selection,
+vignetmaker, ngmix) consumes this artifact unchanged; the module is a drop-in
+replacement for ``mask`` at the flag-image contract.
+
+The same module serves tiles and exposure CCDs: only the input image (hence
+WCS) differs. The healsparse file, its bit meanings, and all flag values live
+**only in config** — the mask products evolve and are swapped at zero code cost.
+
+Module-specific config file entries
+===================================
+
+MASK_PATH : str
+    Path to the healsparse mask file (``.hsp``/``.fits``); band-agnostic
+BIT_FLAG_MAP : str
+    Mapping from healsparse pixel bit value to output flag value, formatted as
+    ``"<bit>:<flag>, <bit>:<flag>, ..."`` (e.g. ``"64:1, 2048:2"``). A pixel
+    carrying several bits receives the bitwise-OR of the mapped flag values.
+OFF_MAP_FLAG : int, optional
+    Flag value assigned to pixels that fall outside the healsparse map
+    footprint (sentinel pixels); default is ``0``. Off-footprint pixels are
+    usually unobserved, so a non-zero value flags them.
+USE_EXT_FLAG : bool, optional
+    If ``True``, sum an external instrument flag file (given as the second
+    input) into the rasterized mask; default is ``False``. Needed for
+    exposures, where saturation / bleeding / bad columns arrive with the data.
+HDU : int, optional
+    HDU of the external instrument flag FITS file; default is ``0``
+PREFIX : str, optional
+    Prefix prepended to the output file name base ``flag``; default is ``""``
+CHUNK_SIZE : int, optional
+    Number of image rows evaluated per chunk; default is chosen from the image
+    size to bound memory (``1e8`` px for tiles, ``1e7`` px for exposure CCDs)
+
+"""
+
+__all__ = ["mask_ext"]