diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index e37fc4d0..ad8f08cc 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -40,6 +40,16 @@ def get_calibrated_quantities(gal_metacal): weights mask : array of bool mask to indicate valid objects in "no-shear" sample + + Notes + ----- + Calibration uses a single global response: ``gal_metacal.R`` is the + 2x2 shear+selection response matrix averaged over the whole run, and + the same matrix is applied to every object. Per-object responses + (``R_g11`` ... ``R_g22``) are computed upstream but are only ever + consumed as catalogue-wide (or, in ``get_w_des`` / + ``get_quantities_binned``, per-bin) means -- no object is calibrated + with its own response. """ # mask for 'no shear' images mask = gal_metacal.mask_dict["ns"] @@ -976,6 +986,14 @@ def _masking_gal(self): self.mask_dict[name] = ind_masked + if len(self.mask_dict["ns"]) == 0: + raise ValueError( + "0 objects survive the metacal selection (no-shear sample" + " is empty: flag == 0 + SNR + size cuts left nothing) --" + " upstream shape measurement likely failed wholesale;" + " check the ngmix outputs before calibrating." + ) + def _masking_gal_mom(self): """Add docstring. diff --git a/workflow/image_sims/Snakefile b/workflow/image_sims/Snakefile index d7a89065..2cf36233 100644 --- a/workflow/image_sims/Snakefile +++ b/workflow/image_sims/Snakefile @@ -17,8 +17,10 @@ command on candide, with the committed SLURM profile owning all scheduling: defaults automatically, so only ``my_run.yaml`` (the science knobs, and any operational override that run wants) is passed on the command line; Snakemake deep-merges the two. The profile supplies the executor, account, partition, -node excludes and job floor -- no ``-j`` needed (the slurm executor sets the -job cap). Off-cluster, drop ``--profile`` and add ``-j N`` to run locally. +node excludes, job cap and job floor -- no ``-j`` needed on the command line +(snakemake >= 9 requires an explicit cap even with the slurm executor, so the +profile carries ``jobs``). Off-cluster, drop ``--profile`` and add ``-j N`` +to run locally. The target (``im_mbias``) is given *before* ``--configfile``: Snakemake's ``--configfile`` takes one-or-more paths, so a target placed after it is diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index 8f719c68..75dd7c43 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -63,6 +63,13 @@ image_sims: # ShapePipe cfis configs (final_cat.param etc.); default is # {shapepipe_repo}/example/cfis_image_sims once #766 lands. [operational] config_dir: /n17data/cdaley/unions/scratch-wf/imsims-run/grids/_cfis_image_sims + # ShapePipe config tree the *pipeline* stage runs against (run_job's -c + # override), the one that owns the ngmix ini and hence METACAL_PSF. Empty -> + # run_job self-computes {shapepipe_repo}/example/cfis_image_sims (the + # historical default, byte-exact). Point it at a per-arm copy carrying a + # different ngmix ini to A/B the metacal PSF without editing the worktree. + # [operational] + pipeline_config_dir: "" psf_model: psfex # [operational] n_smp: -1 # [operational] # Extract/calibrate scripts run from the sp_validation repo checkout (branch diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 0316b909..4f321189 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -35,6 +35,11 @@ executor: slurm +# Snakemake >= 9 requires an explicit job cap even with the slurm executor +# (9.16.3 errors out without one); carrying it here keeps the run command +# free of ``-j`` as the Snakefile docstring promises. +jobs: 100 + # Cluster policy applied to every job unless a rule overrides it. The excludes # are the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, # n36); ``slurm_extra`` is passed verbatim onto the sbatch line by the executor diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index a4eb92d0..6b82ba64 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -86,6 +86,7 @@ _OPERATIONAL_KEYS = { "branches", "shape", "config_dir", + "pipeline_config_dir", "psf_model", "n_smp", "extract_script", @@ -168,6 +169,13 @@ MASK_CONFIG = IMSIM["mask_config"] # e.g. config/calibration/mask_v1.X.9_im_sim PARAMS_TEMPLATE = f"{SPV_REPO}/workflow/image_sims/params_im_sim.py" # ShapePipe cfis_image_sims config dir (per-tile/exposure configs + final_cat.param). CONFIG_DIR = IMSIM["config_dir"] +# ShapePipe config dir the *pipeline* stage's run_job uses (its ``-c`` override, +# the tree that owns the ngmix ini and thus METACAL_PSF). Empty (the default) +# means "let run_job self-compute {shapepipe_repo}/example/cfis_image_sims" -- +# byte-for-byte the historical behaviour, so an unset key reproduces prior runs +# exactly. Set it (e.g. per A/B arm) to point the ShapePipe stage at a config +# tree carrying a different ngmix ini without touching the shared worktree. +PIPELINE_CONFIG_DIR = IMSIM["pipeline_config_dir"] # ShapePipe scripts live in the ShapePipe repo (also baked into its image). CREATE_FINAL_CAT = f"{SHAPEPIPE_REPO}/scripts/python/create_final_cat.py" @@ -372,6 +380,10 @@ rule im_pipeline: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", psf=IMSIM["psf_model"], n_smp=IMSIM["n_smp"], + # ``-c DIR`` only when pipeline_config_dir is set; empty -> run_job + # self-computes its default, so the flag (and its output) is absent for + # a bit-exact reproduction of a run that predates this key. + config_flag=f"-c {PIPELINE_CONFIG_DIR}" if PIPELINE_CONFIG_DIR else "", resources: mem_mb=16000, runtime=720, @@ -379,7 +391,7 @@ rule im_pipeline: "cd {params.run_dir} && " "{EXEC_PIPELINE} bash {RUN_JOB} " "-e {wildcards.tile} -t image_sims -j {JOB_MASK} " - "-p {params.psf} -N {params.n_smp}" + "-p {params.psf} -N {params.n_smp} {params.config_flag}" rule im_merge: