Coverage nexp masks

docs/source/pipeline_canfar.md

@@ -29,7 +29,15 @@ canfar auth switch default
 
 ### Set variables (optional)
 
-Set the current patch in the shell as
+> **Patch convention by catalogue version.** Runs up to `v1.6` are organised
+> in sky patches `P1`–`P9` (7 patches for `v1.3`/`v1.4`, 8 for `v1.5`), each
+> patch a run directory of its own with its `summary/` subdirectory. **The
+> patch concept is removed in `v2.0`**: there is a single run root, and
+> summary files (`summary/missing_job_32_all.txt`, `exp_numbers.txt`) live
+> directly under it. Tools that take a catalogue version (e.g.
+> `get_ccds_with_psf -V`) follow this convention automatically.
+
+For a `v1.x` run, set the current patch in the shell as
 
 ```bash
 patch=P[1-9]
@@ -458,13 +466,12 @@ calibrate_comprehensive
 
 
 ```{note}
-The matched-star-catalogue and coverage-mask diagnostics in the next two
-sections have largely moved out of ShapePipe into
-[`sp_validation`](https://github.com/CosmoStat/sp_validation) / `cosmo_val`.
-Several helpers referenced below — `merge_psf_cat.py`, `download_headers`,
-`extract_field_corners`, `build_coverage_map`, `plot_coverage_map` — are no
-longer shipped in this repository; see `sp_validation` for their current
-equivalents. (`get_ccds_with_psf` and `summary_run` are still part of ShapePipe.)
+The matched-star-catalogue diagnostic (`merge_psf_cat.py`) has moved into
+[`sp_validation`](https://github.com/CosmoStat/sp_validation) / `cosmo_val`;
+see that repository for its current equivalent. The coverage-mask helpers
+(`get_ccds_with_psf`, `download_headers`, `extract_field_corners`,
+`build_coverage_map`, `plot_coverage_map`, and `summary_run`) are shipped by
+ShapePipe and documented below.
 ```
 
 ### Create matched star catalogue
@@ -495,29 +502,57 @@ If the file `$patch/exp_numbers.txt` does not exist for a given patch, create it
 summary_run $patch 1
 ```
 
-Now, create the list of CCDs that have PSF information with
+Now, create the list of single-CCD footprints that have a valid PSF model.
+Only these CCDs are stamped into the mask, so the accumulated map counts, per
+sky pixel, the number of exposures with a valid PSF. The list is written to
+`ccds_with_psf_v1.6.txt`:
 
 ```bash
-get_ccds_with_psf -v -V v1.6
+get_ccds_with_psf -v -V v1.6 -o ccds_with_psf_v1.6.txt
 ```
 
-Next, download exposures headers
+For a (patch-less) `v2.0` run, call this from the run root instead:
 
 ```bash
-download_headers -i ccds_with_psfs_v1.6.txt -o headers_v1.6 -v
+get_ccds_with_psf -v -V v2.0 -o ccds_with_psf_v2.0.txt
 ```
 
-From the headers, the CCD corner coordinates are extracted with
+Next, download the exposure headers; indicate (with `-d`) a directory of
+already downloaded headers, which are linked so duplicate downloads are
+skipped:
+
 ```bash
-extract_field_corners -i headers_v1.6 -v
+download_headers -i ccds_with_psf_v1.6.txt -o headers_v1.6 -d headers_v1.3 -v
+```
+
+From the headers, the per-CCD corner coordinates are extracted. Pass the
+CCD list with `-l` so only CCDs with a valid PSF are written:
+
+```bash
+extract_field_corners -i headers_v1.6 -l ccds_with_psf_v1.6.txt -o exp_ra_dec_v1.6.txt -v
 ```
 
 Then, build the healsparse coverage mask file as
 ```bash
-build_coverage_map
+build_coverage_map -i exp_ra_dec_v1.6.txt -o coverage_v1.6.x.hsp -c 128 -n 131072 -v
 ```
 
-Use `plot_coverage_map` to create plots of the coverage mask.
+The healsparse resolutions (128, 131072) match the bit masks.
+
+
+Use `plot_coverage_map` to plot a region of the coverage mask. For example,
+the SGC region with a colorbar clipped to the 1–5 exposure range:
+
+```bash
+plot_coverage_map -i coverage_v1.6.x.hsp -o coverage_v1.6_SGC.png \
+    -C -R -20 -r 45 -D 18 -d 40 -m 1 -M 5 -v
+```
+
+Here `-R`/`-r` and `-D`/`-d` set the RA and Dec plot limits, `-m`/`-M` the
+colorbar range, and `-C` adds the colorbar. Building and plotting for a range
+of versions (SGC and NGC regions) is automated by
+`build_and_plot_coverage_maps.sh`.
+
 
 ## Extra Utilities
 

log_-i

@@ -0,0 +1 @@
+-i /tmp/pytest-of-cdaley/pytest-8/test_build_map_pole_guard_skip0/corners.txt -o /tmp/pytest-of-cdaley/pytest-8/test_build_map_pole_guard_skip0/cov.hsp -c 32 -n 1024

pyproject.toml

@@ -24,6 +24,8 @@ dependencies = [
     "cs_util>=0.2.1",
     "galsim>=2.8",
     "h5py",
+    "healsparse",
+    "hpgeom",
     "joblib>=1.4",
     "matplotlib>=3.10",
     "mccd>=1.2.4",
@@ -39,6 +41,7 @@ dependencies = [
     "reproject>=0.19",
     "sf_tools>=2.0.4",
     "skaha>=1.7",
+    "skyproj",
     "sqlitedict>=2.0",
     "termcolor",
     "tqdm>=4.68.1",
@@ -63,6 +66,10 @@ jupyter = [
 lint = [
     "ruff",
 ]
+plot = [
+    "skyproj",
+    "matplotlib>=3.10",
+]
 release = [
     "build",
     "twine",
@@ -73,14 +80,19 @@ test = [
     "pytest-cov>=5.0",
 ]
 fitsio = ["fitsio"]
-dev = ["shapepipe[doc,jupyter,lint,release,test,fitsio]"]
+dev = ["shapepipe[doc,jupyter,lint,plot,release,test,fitsio]"]
 
 [project.scripts]
 shapepipe_run       = "shapepipe.shapepipe_run:main"
 summary_run         = "shapepipe.summary_run:main"
 canfar_submit_job   = "shapepipe.canfar_run:run_job"
 canfar_monitor      = "shapepipe.canfar_run:run_log"
 canfar_monitor_log  = "shapepipe.canfar_run:run_monitor_log"
+get_ccds_with_psf   = "shapepipe.get_ccds_run:run_ccd_psf_handler"
+download_headers    = "shapepipe.coverage_run:run_download_headers"
+extract_field_corners = "shapepipe.coverage_run:run_extract_corners"
+build_coverage_map  = "shapepipe.coverage_run:run_build_coverage"
+plot_coverage_map   = "shapepipe.coverage_run:run_plot_coverage"
 
 [tool.uv]
 # shapepipe targets Linux only; skip Windows/macOS wheel resolution

scripts/python/get_ccds_with_psf.py

@@ -9,192 +9,50 @@
 
 """
 
-
 import sys
-import numpy as np
-
-from shapepipe.utilities import summary
-
-
-def get_lines(fname):
-    """Get Lines.
-
-    Return list of lines read from a text file.
-
-    Parameters
-    ----------
-    fname: str
-        input file name
 
-    Returns:
-    list
-        IDs
+from shapepipe.utilities.ccd_psf_handler import CcdPsfHandler
 
-    """
-    IDs = []
-    with open(fname) as f:
-        lines = f.readlines()
-        for line in lines:
-            IDs.append(line.rstrip())
-
-    return IDs
 
+def run_ccd_psf_handler(args=None):
+    """Run CCD PSF Handler.
 
-def get_exp_shdu_missing(patches):
-    """Get Exp Shdu Missing.
-
-    Returns set of missing CCDs (single-exposure single-HDU IDs) from a list of patches.
+    Create instance and run the CCD PSF handler.
 
     Parameters
     ----------
-    patches: list
-        input patches
+    args : list, optional
+        command line arguments
 
     Returns
     -------
-    set
-       missing CCD IDs
+    int
+        exit code
 
     """
-    exp_shdu_missing_all = set()
-
-    for patch in patches:
-
-        path_exp_shdu_missing = f"{patch}/summary/missing_job_32_all.txt"
-        exp_shdu_missing = get_lines(path_exp_shdu_missing)
-
-        print(f"Patch {patch}: Found {len(exp_shdu_missing)} missing ccds", end="; ")
-
-        exp_shdu_missing_all.update(exp_shdu_missing)
-
-        print(f"cumulative {len(exp_shdu_missing_all)} missing ccds")
-
-    print()
-
-    return exp_shdu_missing_all
-
-
-def get_exp(patches):
-    """Get Exp.
-
-    Return set of exposures from a list of patches.
-
-    Parameters
-    ----------
-    patches: list
-        input patches
-
-    Returns
-    -------
-    set
-       exposure IDs
-
-    """
-    exp_all = set()
-
-    for patch in patches:
-
-        path_exp = f"{patch}/exp_numbers.txt"
-        exp = get_lines(path_exp)
-
-        print(f"Patch {patch}: Found {len(exp)} exposures", end="; ")
-
-        exp_all.update(exp)
-
-        print(f"cumulative {len(exp_all)} exposures")
+    # Create instance
+    obj = CcdPsfHandler()
 
-    print()
+    return obj.run(args=args)
 
-    return exp_all
 
+def main(argv=None):
+    """Main.
 
-def get_ccds_with_psf(patches, n_CCD=40):
-    """Get CCDs With PSF.
-
-    Return set of CCDs from list of patches.
+    Main program.
 
     Parameters
     ----------
-    patches: list
-        input patches
+    argv : list, optional
+        command line arguments
 
     Returns
     -------
-    set
-       CCD IDs
+    int
+        exit code
 
     """
-    # Get missing CCDs
-    print("=== get missing CCDs ===")
-    exp_shdu_missing_all = get_exp_shdu_missing(patches)
-
-    # Get all exposures used in tiles
-    print("=== get exposures ===")
-    exp_all = get_exp(patches)
-
-    # Turn exposures into exposure-single-HDU names (CCDs)
-    exp_shdu_all = summary.get_all_shdus(exp_all, n_CCD)
-
-    print(f"Found {len(exp_shdu_all)} CCDs")
-
-    return exp_shdu_all
-
-
-def get_ccds_with_psf_method_2(patches, n_CCD=40):
-
-    for patch in patches:
-        directory = f"{patch}/exp_runs"
-
-    for entry in os.scandir(directory):
-        pass
-
-def save(IDs, path):
-    """Save.
-
-    Save list of IDs to text file.
-
-    Parameters
-    ----------
-    IDs: set
-        input IDs
-
-    path: str
-        output file name
-
-    """
-    with open(path, "w") as f_out:
-        for ID in IDs:
-            print(ID, file=f_out)
-
-def main(argv):
-    """Main.
-
-    Main program.
-
-    """
-    version = "v1.5"
-
-    if version == "v1.4":
-        n_patch = 7
-    elif version == "v1.5":
-        n_patch = 8
-    else:
-        raise ValueError(f"Invalid version {version}")
-
-    patches = [f'P{x}' for x in np.arange(n_patch) + 1]
-
-    print(f"=== get_ccds_with_psf for version {version}, patches {patches} ===")
-
-    print("=== method 1: exp_list - missing === ")
-    exp_shdu_all = get_ccds_with_psf(patches)
-
-    save(exp_shdu_all, f"ccds_with_psf_{version}.txt")
-
-    #print("=== method 2: star cats === ")
-    #exp_shdu_all_method_2 = get_ccds_with_psf_method_2(patches)
-    #save(exp_shdu_all_method_2, f"ccds_with_psf_{version}_method_2.txt")
-
-    return 0
+    return run_ccd_psf_handler(args=argv)
 
 
 if __name__ == "__main__":