Persist per-exposure PSF products before reclamation (exp_persist)

workflow/README.md

@@ -154,7 +154,7 @@ workflow/
   bin/sp                 committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel)
   rules/
     prepare.smk          tile get_images/uncompress/find_exposures
-    exposure.smk         per-exposure: get_images, star_cat, split, mask, psf (no temp())
+    exposure.smk         per-exposure: get_images, star_cat, split, mask, psf, persist (no temp())
     tile.smk             per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat
   scripts/
     sp_rule.py           the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor)
@@ -163,6 +163,7 @@ workflow/
     completeness.py      the ported count-floor table (shared by sp_rule + run_report)
     run_report.py        standalone report (NOT a DAG node; run_report hooks call it)
     container.py         image layers + the resolution order behind `sp container` (stdlib-only)
+    persist_exp.py       ONE exposure's keepable PSF products -> one tar on products_dir (the exp_persist rule)
     clean_exposure.py    ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule)
 profiles/nibi/config.yaml  SLURM executor; apptainer SDM; per-user jobs cap; keep-going
 ```
@@ -230,6 +231,22 @@ profiles/nibi/config.yaml  SLURM executor; apptainer SDM; per-user jobs cap; kee
   trigger reads that cut as a reason to rerun the very tiles it protects.
   Know the consequence — `--forcerun` on a tile whose `final_cat` exists will
   not rebuild its reclaimed exposures. Delete the `final_cat` first.
+- **PSF products leave scratch before the purge does.** `exp_persist` packs
+  the files named by `persist_exp:` in `config.yaml` (default: the psfex_interp
+  `validation_psf-*.fits`, the rho/tau statistics input) from the exposure's
+  scratch store into ONE uncompressed tar,
+  `<products_dir>/exp/<prefix>/<base>/psf/<base>.tar` (inodes, not bytes, bind
+  on /project), and writes ONE manifest beside it recording the patterns, the
+  members and their sizes. The
+  threat it answers is the /scratch purge, not `clean_exposure` — the store goes
+  in 60 days whether or not the workflow reclaimed it — so it runs even with
+  `clean: false`, requested directly by `rule all`. `clean_exposure` takes its
+  manifest as an input, so reclamation can never overtake the copy. It is a
+  rule of its own rather than a `cp` on the end of `exp_psf` because the keep
+  list rides on `params`: adding a pattern reruns seconds of packing, not four
+  hours of PSF fitting per exposure. A pattern that matches nothing is a
+  recorded warning (setools rejects sparse CCDs); matching nothing at all is a
+  failure. A `localrule`, like `exp_star_cat` and for the same arithmetic.
 - **A dead tile can be told to stop pinning exposures.** An exposure is
   cleanable only once every consuming tile has its vignets, so one
   permanently-failed tile holds its ~80 exposures for the life of the

workflow/Snakefile

@@ -258,6 +258,7 @@ EXP_DIR  = str(RUN_DIR / "exp" / "{shard}" / "{exp}")
 # The persistent root mirrors the scratch one, shard for shard, so the two trees
 # read as the same campaign seen from two filesystems.
 PROD_TILE_DIR = str(PRODUCTS_DIR / "tiles" / "{shard}" / "{tile}")
+PROD_EXP_DIR  = str(PRODUCTS_DIR / "exp" / "{shard}" / "{exp}")
 
 def tile_dir(tile):
     return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}"
@@ -271,6 +272,18 @@ def tile_manifest(tile, stage):
 def exp_manifest(exp, stage):
     return f"{exp_dir(exp)}/manifests/{stage}.json"
 
+def prod_exp_dir(exp):
+    """The exposure's dir on the PERSISTENT root — where exp_persist writes.
+
+    Sharded identically to the scratch one, so the two trees read as the same
+    campaign seen from two filesystems, exposure side as well as tile side."""
+    return f"{PRODUCTS_DIR}/exp/{exp[:2]}/{exp}"
+
+def prod_exp_manifest(exp, stage):
+    """A manifest that must SURVIVE reclamation, so it is not in the exposure's
+    scratch manifests/ dir (clean_exposure deletes that wholesale)."""
+    return f"{prod_exp_dir(exp)}/manifests/{stage}.json"
+
 def forest_dir(tile):
     return f"{tile_dir(tile)}/exp_forest"
 
@@ -309,6 +322,7 @@ SCRIPT_HASH = script_hash("completeness.py")
 FOREST_HASH = script_hash("build_forest.py")
 CLEAN_HASH  = script_hash("clean_exposure.py")
 CLEAN_TILE_HASH = script_hash("clean_tile.py")
+PERSIST_HASH = script_hash("persist_exp.py")
 # Same argument for star_cats.py, which both star-cat rules call: their params
 # otherwise fingerprint nothing but paths, so an edit to the chunking or the cut
 # would never rerun them. ONE hash for both rules because it is one script — and
@@ -434,6 +448,40 @@ def clean_targets():
             out.append(tombstone(exp))
     return sorted(out)
 
+# --- persisted exposure products (D5) --------------------------------------
+# The keep list is config, not a rule input, and it is READ HERE so that exactly
+# one place converts it into the form the rule carries. An empty list is a
+# deliberate "keep nothing" and produces no jobs at all.
+PERSIST_EXP = list(config.get("persist_exp") or [])
+
+
+def persist_targets():
+    """Which exposures this invocation must pack PSF products off scratch for.
+
+    `rule all` requests these DIRECTLY rather than reaching them only through
+    clean_exposure. Persistence and reclamation are different concerns — the
+    /scratch purge takes the store whether or not `clean:` is on — and hanging
+    the copy off the clean rule alone would mean a campaign run with clean:false
+    persists nothing and loses everything at the purge.
+
+    Scope is the ready tiles' exposures, which `all` already builds through the
+    tile chain, so nothing new is pulled into the DAG by asking.
+
+    EXCEPT A CLEANED EXPOSURE. Its exp_psf manifest was deleted by
+    clean_exposure, so requesting its persist manifest would make the DAG
+    rebuild the whole exposure chain from VOS — the avalanche tile.smk's
+    reclaimed-edge cut exists to prevent, arriving through a new target instead.
+    A tombstone means the copy already happened (clean_exposure cannot run
+    before exp_persist), so there is nothing to ask for.
+
+    HEAD PROCESS ONLY, for the same reason as clean_targets() above.
+    """
+    if not PERSIST_EXP or not workflow.is_main_process:
+        return []
+    exps = {e for t in TILES_READY for e in tile_exposures(t)}
+    return sorted(prod_exp_manifest(e, "exp_persist") for e in exps
+                  if not Path(tombstone(e)).exists())
+
 # --- tile reclamation (D5) --------------------------------------------------
 # A SEPARATE FLAG from `clean:`, deliberately (config.yaml carries the full
 # argument): exposure reclamation costs nothing but a rebuild if a tile is
@@ -589,22 +637,29 @@ include: "rules/exposure.smk"
 include: "rules/tile.smk"
 
 # --- top-level targets ------------------------------------------------------
-# The aggregation targets, clean_exposure, clean_tile, star_catalogue and
-# exp_star_cat run in the head process. The two clean rules are seconds of rmtree
+# The aggregation targets, clean_exposure, clean_tile, star_catalogue,
+# exp_star_cat and exp_persist run in the head process. The two clean rules are seconds of rmtree
 # and hang off `all`; exp_star_cat is seconds of local FITS work; all three would
 # otherwise be ~20k (clean_exposure, exp_star_cat) or ~23k (clean_tile) sbatch
 # submissions at DR6 scale for work shorter than the scheduling latency.
 # star_catalogue is one job either way, and local keeps its CDS concurrency the
 # explicit number its thread pool sets (see exposure.smk).
 #
+# exp_persist joins them for the same arithmetic — a few MB of `tar` per exposure,
+# ~20k of them at DR6 scale, each far shorter than the scheduling latency that
+# would submit it (exposure.smk argues the placement in full).
+#
 # star_catalogue and exp_star_cat are MID-CHAIN localrules, so they must stay out
 # of any future `group:` label: a local job cannot be fused into a submitted group.
-# The two clean rules are DAG leaves and have no such constraint.
-localrules: all, prepare_all_tiles, clean_exposure, clean_tile, star_catalogue, exp_star_cat
+# exp_persist sits between exp_psf and clean_exposure, both of which are outside
+# every group already (exp_psf is heavy, clean_exposure is local), so it adds no
+# new constraint. The two clean rules are DAG leaves and have none either.
+localrules: all, prepare_all_tiles, clean_exposure, clean_tile, star_catalogue, exp_star_cat, exp_persist
 
 rule all:
     input:
         [final_cat(t) for t in TILES_READY],
+        persist_targets(),
         clean_targets(),
         clean_tile_targets(),
 

workflow/config.yaml

@@ -83,6 +83,78 @@ star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache
 # would otherwise have to rebuild from tile headers.
 index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite
 
+# Per-exposure PSF products to carry onto the persistent root before the scratch
+# store goes (`exp_persist`, exposure.smk). A list of plain file-name globs,
+# matched recursively under the PSF chain's four module output dirs
+# (<run_dir>/exp/<shard>/<exp>/output/run_sp_exp_SxSePsfPi/*/output/ —
+# sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner).
+# Matches are packed, flat, into ONE uncompressed tar per exposure:
+# <products_dir>/exp/<shard>/<exp>/psf/<exp>.tar, with a manifest listing the
+# members beside it. One tar rather than loose copies because inodes, not bytes,
+# bind on /project (~1 M-file group quota; loose copies would be ~200 files per
+# exposure, ~2 M at DR6 scale). FITS members read straight from the tar:
+# fits.open(io.BytesIO(tarfile.open(t).extractfile(m).read())).
+#
+# WHY COPY RATHER THAN EXEMPT THESE FROM CLEANUP. Reclamation is not the threat.
+# run_dir is /scratch and is PURGED on a 60-day window whether or not
+# clean_exposure ever ran; products_dir is /project, backed up and not purged.
+# The only way a per-exposure product outlives its campaign is to leave the
+# filesystem. (Ordering is free: clean_exposure takes the exp_persist manifest
+# as an input, so a store is never reclaimed before its keepers are written.)
+#
+# EDITING THIS LIST IS CHEAP. It rides on exp_persist's `params`, so a change
+# reruns the packing (seconds) and NOT exp_psf (four hours per exposure). That
+# separation is the whole reason exp_persist is a rule of its own.
+#
+# The default is the minimum: the psfex_interp VALIDATION catalogue, one per
+# CCD, which is the input to the rho/tau statistics. Without it the PSF
+# diagnostics cannot be recomputed after a purge without rebuilding the exposure
+# chain from VOS.
+#
+# OPT-IN CANDIDATES, and what each buys. Sizes are per exposure (40 CCDs),
+# measured on smk-m2 (127 exposures, 64 tiles); a 64-tile campaign with all of
+# the measured ones on came to 7.2 GB:
+#   validation_psf-*.fits       (the default)                        2.0 MB
+#   *.psf                       the PSFEx model itself. Keeping it means the PSF
+#                               can be re-interpolated at ANY position later
+#                               without rebuilding the exposure chain — the
+#                               single most capability-adding entry here.
+#                                                                    2.8 MB
+#   psfex_cat-*.cat             PSFEx's own output catalogue (FITS_LDAC): the
+#                               per-star FLAGS_PSF / CHI2_PSF, i.e. WHICH stars
+#                               outlier rejection clipped. Not recoverable from
+#                               anything else (the .psf header keeps only the
+#                               LOADED/ACCEPTED counts).           unmeasured
+#   star_selection-*.fits       the PRE-SPLIT selection (setools writes it under
+#                               mask/). The only file that can answer "which
+#                               stars were rejected by the selection cuts, and
+#                               why" — the split samples have already lost the
+#                               rejects.                            24.5 MB
+#   star_split_ratio_80-*.fits  setools' 80% TRAINING star sample, the set PSFEx
+#                               actually fitted. Rows duplicate star_selection.
+#                                                                   19.9 MB
+#   star_split_ratio_20-*.fits  the 20% VALIDATION sample — the positions the
+#                               validation_psf rows correspond to. Rows
+#                               duplicate star_selection.            7.1 MB
+#   star_stat-*.txt             setools' per-CCD STAT block (star counts,
+#                               stars/deg^2, FWHM mode and cuts, under stat/):
+#                               the selection's summary without its catalogue.
+#                                                                  unmeasured
+# A production keep list is `validation_psf` + `*.psf` + `psfex_cat` (~5 MB per
+# exposure); the star_split files are only worth it if star_selection is off.
+# PSFEx residual/check images and its XML diagnostics are NOT candidates as the
+# chain stands: the committed default.psfex sets CHECKIMAGE_TYPE NONE and
+# WRITE_XML N, so nothing is emitted to match. They are a config change first,
+# a pattern second.
+#
+# NOTE ON products_dir DEFAULTING TO run_dir (a fixture or smoke test): the tar
+# then lands beside the store on the same filesystem and buys nothing, and the
+# manifest sits in the exposure's own manifests/ dir, which clean_exposure
+# deletes wholesale — so a one-root run re-persists after every reclamation.
+# Harmless, and exactly the pre-D5 behaviour a one-root run asks for.
+persist_exp:
+  - validation_psf-*.fits
+
 # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one
 # `clean_exposure` job per exposure. It fires once every campaign tile that reads
 # that exposure has its vignets, deletes the exposure's store AND its manifests,

workflow/rules/exposure.smk

@@ -1,6 +1,6 @@
 """Exposure chain — per exposure, keyed by exp base id (dedup is structural).
 
-    exp_get_images -> exp_split -----> exp_mask -> exp_psf
+    exp_get_images -> exp_split -----> exp_mask -> exp_psf -> exp_persist
                    -> exp_star_cat --/
     star_catalogue ---------------/
 
@@ -12,6 +12,13 @@ reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a
 run log. There is no `prepare_exposures` aggregation target: these chains hang
 off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests).
 
+``exp_persist`` is the one rule here that writes to the PERSISTENT root: it
+packs the PSF products named by `persist_exp:` into one tar per exposure off
+/scratch before the purge (or clean_exposure) can take them. It is a separate
+rule from exp_psf precisely so that editing that list costs a `tar` and not a
+four-hour refit; the full
+argument is in workflow/scripts/persist_exp.py.
+
 NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by
 construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN,
 not over one invocation — reclamation here is clean_exposure's job (S5), driven
@@ -315,6 +322,59 @@ rule exp_psf:
         sp_shell("exp_psf", "config_exp_psfex.ini")
 
 
+# --- persistence (D5) -------------------------------------------------------
+# The counterpart of reclamation, and it must come first in the DAG: this packs
+# the exposure's keepable PSF products into one tar on the persistent root, and
+# clean_exposure below takes its manifest as an input so the store is never
+# reclaimed before the keepers have left /scratch. The purge would take them
+# anyway — that, not clean_exposure, is what this rule exists for
+# (persist_exp.py's docstring argues both halves, and config.yaml's
+# `persist_exp:` block carries the keep list and its candidates).
+#
+# A LOCALRULE (declared in the Snakefile), by exactly the arithmetic that made
+# exp_star_cat one: the body is a `tar` of a few MB from one shared filesystem
+# to another, seconds of work, and one sbatch per exposure would be ~20k
+# submissions at DR6 scale for jobs shorter than the scheduling latency. The
+# grouping constraint that binds mid-chain localrules (this file's docstring)
+# does not bite here: exp_persist's only neighbours are exp_psf, which is too
+# heavy to ever fuse, and clean_exposure, which is local itself.
+#
+# ONE DECLARED OUTPUT, AND IT IS A MANIFEST, NOT THE TAR OR A directory(). The
+# tar is not declared: a directory output would attest that a directory exists,
+# where what we want written down is WHICH files were packed and how big each was —
+# the provenance a rho-statistics run months from now needs in order to know
+# what it is reading. The manifest is byte-stable, so a no-op rerun does not
+# move its mtime and does not make clean_exposure look out of date.
+#
+# THE KEEP LIST RIDES ON params. That is the entire reason this is not three
+# lines of tar appended to exp_psf's shell: `params` is a rerun trigger, so
+# adding a pattern reruns the packing and leaves the PSF chain alone.
+rule exp_persist:
+    input:
+        rules.exp_psf.output.manifest
+    output:
+        manifest = f"{PROD_EXP_DIR}/manifests/exp_persist.json"
+    # No `log:`: the script's only failure modes are "nothing matched" and a
+    # name collision, both of which it reports on stderr and neither of which
+    # has a per-CCD verdict worth a completeness record.
+    params:
+        patterns    = " ".join(f"--pattern '{p}'" for p in PERSIST_EXP),
+        exp_dir     = lambda wc: exp_dir(wc.exp),
+        dest        = lambda wc: f"{prod_exp_dir(wc.exp)}/psf",
+        script_hash = PERSIST_HASH
+    threads: 1
+    retries: 2
+    resources:
+        mem_mb = 2000,
+        runtime = 10
+    shell:
+        "set -euo pipefail\n"
+        f"python {SCRIPTS}/persist_exp.py"
+        " --exp-dir '{params.exp_dir}' --exp {wildcards.exp}"
+        " --dest '{params.dest}' --manifest {output.manifest}"
+        " {params.patterns}"
+
+
 # --- reclamation (D5) -------------------------------------------------------
 # The one exception to "no reclamation in this file": clean_exposure OWNS
 # exposure-level deletion, and it is a real job, not temp() bookkeeping, because
@@ -346,7 +406,16 @@ rule clean_exposure:
         # spatial neighbours. In-scope consumers keep their edge: they may run in
         # this DAG, so the clean must be ordered after them.
         lambda wc: [tile_manifest(t, "tile_vignets")
-                    for t in clean_consumers(wc.exp) if t in READY_SET]
+                    for t in clean_consumers(wc.exp) if t in READY_SET],
+        # The keepers must be off /scratch before the store goes. Unlike the
+        # consumer edges above, this edge does not depend on scope: it is the
+        # same exposure's own rule, so it drags nothing into the DAG that this
+        # exposure's chain did not already put there. It is conditional only on
+        # there being a keep list at all — with `persist_exp:` empty, "keep
+        # nothing" is a coherent instruction and must not become a dependency on
+        # a rule that would fail for having nothing to copy.
+        lambda wc: ([prod_exp_manifest(wc.exp, "exp_persist")]
+                    if PERSIST_EXP else [])
     output:
         tombstone = f"{EXP_DIR}/cleaned.json"
     params: