From 89cdf53e3c397683c25b6dc0ba05f741c7143463 Mon Sep 17 00:00:00 2001 From: Dmitrijs Lvovs Date: Mon, 3 Aug 2026 13:26:24 -0400 Subject: [PATCH 1/9] fix writing to readonly path inside container --- main.nf | 5 +++-- nextflow.config | 31 ++++++++++++++++++++++--------- pipeline.py | 13 ++++++++++--- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/main.nf b/main.nf index 954ab2b..ac821e3 100644 --- a/main.nf +++ b/main.nf @@ -1,6 +1,7 @@ process INTERPRET { tag "${report.baseName}" - container 'llmize:latest' + container 'ghcr.io/fertiglab/llmize:latest' + containerOptions "-v ${projectDir}:/opt/llmize:ro" publishDir params.outdir, mode: 'copy' input: @@ -25,12 +26,12 @@ process INTERPRET { def top_k_flag = params.top_k != null ? "--top_k ${params.top_k}" : '' def numpred_flag = params.num_predict != null ? "--num_predict ${params.num_predict}" : '' """ - ${boot} STAMP=\$(date +%Y%m%d_%H%M%S) python3 ${home}/pipeline.py \\ --input '${report}' \\ --model '${params.model}' \\ --num_ctx ${params.num_ctx} \\ + --work-dir . \\ ${think_flag} ${enrich_flag} ${review_flag} ${whole_flag} ${synth_flag} \\ ${prompt_flag} ${temp_flag} ${seed_flag} ${top_p_flag} ${top_k_flag} ${numpred_flag} \\ --output "${report.baseName}_interpretation_\${STAMP}.md" diff --git a/nextflow.config b/nextflow.config index 6004263..8d93919 100644 --- a/nextflow.config +++ b/nextflow.config @@ -18,14 +18,33 @@ params { num_predict = null } -docker { - enabled = true -} profiles { + docker { + docker.enabled = true + } native { docker.enabled = false } + igs { + apptainer { + enabled = true + autoMounts = true + pullTimeout = '60m' + } + params { + slurm_account = null + } + process { + executor = 'slurm' + clusterOptions = { + def account = params.slurm_account?.toString()?.trim() + account ? "-A ${account}" : '' + } + errorStrategy = 'retry' + maxRetries = 3 + } + } } report { @@ -40,9 +59,3 @@ timeline { overwrite = true } -process { - withName: INTERPRET { - container = 'llmize:latest' - containerOptions = "-v ${System.getProperty('user.home')}/.llmize-ollama:/root/.ollama" - } -} diff --git a/pipeline.py b/pipeline.py index bcfc5a0..fa9433a 100644 --- a/pipeline.py +++ b/pipeline.py @@ -125,6 +125,11 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Run environment preflight checks (Ollama, models, ToolUniverse, schema) and exit.", ) + parser.add_argument( + "--work-dir", + default=".", + help="Working directory for saving intermediate and final output files (default: current directory).", + ) return parser.parse_args() @@ -167,6 +172,7 @@ def run_pipeline( annotated_filename: str | None, output_path: str | None, num_ctx: int, + work_dir: str = ".", whole_report: bool = False, enrich: bool = False, synthesize_final: bool = True, @@ -184,7 +190,7 @@ def run_pipeline( if extracted_filename is None: extracted_filename = default_output_name(input_path, prefix="extracted_") - extracted_path = save_json(reduced, DATA_DIR, extracted_filename) + extracted_path = save_json(reduced, work_dir, extracted_filename) print(f"[pipeline] Extracted JSON saved: {extracted_path}") if annotated_filename is None: @@ -196,7 +202,7 @@ def run_pipeline( annotated_path = merge( data_path=extracted_path, descriptor_path=descriptor_path, - output_dir=DATA_DIR, + output_dir=work_dir, output_filename=annotated_filename, focal_labels=focal_labels, ) @@ -238,7 +244,7 @@ def run_pipeline( timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") stem = os.path.splitext(os.path.basename(input_path))[0] output_filename = f"{stem}_interpretation_{timestamp}.md" - output_path = os.path.join(DATA_DIR, output_filename) + output_path = os.path.join(work_dir, output_filename) footer = build_run_footer( model=model, num_ctx=num_ctx, think=think, gen_options=gen_options, @@ -268,6 +274,7 @@ def main() -> None: annotated_filename=args.annotated_output, output_path=args.output, num_ctx=args.num_ctx, + work_dir=args.work_dir, whole_report=args.whole_report, enrich=args.enrich, synthesize_final=not args.no_synthesis, From 6ccd297066f5b5766a689666115a32694d34b31e Mon Sep 17 00:00:00 2001 From: Dmitrijs Lvovs Date: Mon, 3 Aug 2026 19:26:55 -0400 Subject: [PATCH 2/9] WIP but not yet yet failed --- main.nf | 12 +++++++++++- nextflow.config | 12 ++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/main.nf b/main.nf index 244988f..f1b87c0 100644 --- a/main.nf +++ b/main.nf @@ -1,6 +1,9 @@ process INTERPRET { tag "${report.baseName}" - container 'ghcr.io/fertiglab/llmize:latest' + label "process_gpu" + resourceLimits cpus: 4, memory: 24.GB, time: '1h' + //container 'ghcr.io/fertiglab/llmize:latest' + container 'ghcr.io/fertiglab/llmize:sha-f2ae922' publishDir params.outdir, mode: 'copy' input: @@ -24,6 +27,13 @@ process INTERPRET { def top_k_flag = params.top_k != null ? "--top_k ${params.top_k}" : '' def numpred_flag = params.num_predict != null ? "--num_predict ${params.num_predict}" : '' """ + export HOME="\$PWD" + export XDG_CACHE_HOME="\$PWD/.cache" + export OLLAMA_MODELS="\$PWD/ollama/models" + mkdir -p "\$OLLAMA_MODELS" "\$XDG_CACHE_HOME" + + ${boot} + STAMP=\$(date +%Y%m%d_%H%M%S) python3 ${home}/pipeline.py \\ --input '${report}' \\ diff --git a/nextflow.config b/nextflow.config index cc21c76..9d5960d 100644 --- a/nextflow.config +++ b/nextflow.config @@ -37,11 +37,19 @@ profiles { process { executor = 'slurm' clusterOptions = { - def account = params.slurm_account?.toString()?.trim() - account ? "-A ${account}" : '' + account ? "-A ${params.slurm_account?.toString()?.trim()}" : '' } errorStrategy = 'retry' maxRetries = 3 + withLabel: 'process_gpu' { + // Pass Slurm GRES arguments to allocate hardware + clusterOptions = "--gres=gpu:1 -A ${params.slurm_account?.toString()?.trim()}" + + // Allocate matching CPU/RAM to feed the GPU + cpus = 4 + memory = '32 GB' + time = '2h' + } } } } From 09e0e3d506f5bc14efd7396afbb1916cb9f133fe Mon Sep 17 00:00:00 2001 From: Dmitrijs Lvovs Date: Tue, 4 Aug 2026 13:31:04 -0400 Subject: [PATCH 3/9] first working version --- nextflow.config | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nextflow.config b/nextflow.config index 9d5960d..d76a7d7 100644 --- a/nextflow.config +++ b/nextflow.config @@ -42,13 +42,15 @@ profiles { errorStrategy = 'retry' maxRetries = 3 withLabel: 'process_gpu' { - // Pass Slurm GRES arguments to allocate hardware - clusterOptions = "--gres=gpu:1 -A ${params.slurm_account?.toString()?.trim()}" + // Allocate a gpu + clusterOptions = "--gres=gpu:1 -A ${params.slurm_account?.toString()?.trim()}" + // Enable Nvidia in apptainer + containerOptions = "--nv" // Allocate matching CPU/RAM to feed the GPU - cpus = 4 + cpus = 2 memory = '32 GB' - time = '2h' + time = '4h' } } } From 9c0d3044aea6b3a774d721909709e0da0d0906cd Mon Sep 17 00:00:00 2001 From: Dmitrijs Lvovs Date: Tue, 4 Aug 2026 17:46:04 -0400 Subject: [PATCH 4/9] persist models across runs --- README.md | 34 ++++++++++++++++++++++++++++++++++ main.nf | 4 +++- nextflow.config | 9 +++++---- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b5d9cf6..418b13f 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,40 @@ python3 pipeline.py --check It prints a clear ✓/⚠/✗ report and exits non-zero if a required check fails. Optional features (e.g. ToolUniverse enrichment) only produce warnings, not failures. +## Nextflow execution + +The Nextflow module uses an Ollama model cache directory via `OLLAMA_MODELS`. + +- In the `igs` and `igs_cpu` profiles, the default cache path is user-specific scratch: + `/usr/local/scratch/$USER/ollama/models` +- On the first run, if the model is missing, the workflow auto-pulls it. +- On later runs, the same model is reused from scratch and pull is skipped. + +### GPU (Slurm + Apptainer) + +```bash +nextflow run main.nf \ + -profile igs \ + --input data/multiqc_data.json \ + --slurm_account \ + -w /usr/local/scratch/$USER/work \ + -resume +``` + +### Override cache path (optional) + +Use this if your cluster requires a different location: + +```bash +nextflow run main.nf \ + -profile igs \ + --input data/multiqc_data.json \ + --slurm_account \ + --ollama_models_dir /path/to/persistent/models \ + -w /usr/local/scratch/$USER/work \ + -resume +``` + ## Continuous integration The GitHub Actions workflow (`.github/workflows/test.yml`) runs on every pull request diff --git a/main.nf b/main.nf index f1b87c0..34731d4 100644 --- a/main.nf +++ b/main.nf @@ -15,6 +15,8 @@ process INTERPRET { script: def home = workflow.containerEngine ? '/opt/llmize' : "${projectDir}" def boot = workflow.containerEngine ? "export LLMIZE_MODEL='${params.model}'\n bash ${home}/docker/boot_ollama.sh" : '' + def ollama_models_escaped = params.ollama_models_dir ? params.ollama_models_dir.toString().replace("'", "'\"'\"'") : null + def ollama_models_export = ollama_models_escaped ? "export OLLAMA_MODELS='${ollama_models_escaped}'" : 'export OLLAMA_MODELS="\$PWD/ollama/models"' def think_flag = "${params.think}".toBoolean() ? '--think' : '--no-think' def review_flag = "${params.review}".toBoolean() ? "--review --review-passes ${params.review_passes}" : '' def whole_flag = "${params.whole_report}".toBoolean() ? '--whole-report' : '' @@ -29,7 +31,7 @@ process INTERPRET { """ export HOME="\$PWD" export XDG_CACHE_HOME="\$PWD/.cache" - export OLLAMA_MODELS="\$PWD/ollama/models" + ${ollama_models_export} mkdir -p "\$OLLAMA_MODELS" "\$XDG_CACHE_HOME" ${boot} diff --git a/nextflow.config b/nextflow.config index d76a7d7..7b8256f 100644 --- a/nextflow.config +++ b/nextflow.config @@ -15,6 +15,7 @@ params { top_p = null top_k = null num_predict = null + ollama_models_dir = null } @@ -44,8 +45,8 @@ profiles { withLabel: 'process_gpu' { // Allocate a gpu clusterOptions = "--gres=gpu:1 -A ${params.slurm_account?.toString()?.trim()}" - // Enable Nvidia in apptainer - containerOptions = "--nv" + // Enable Nvidia in apptainer and mound ollama dir + containerOptions = "--nv -B ${params.ollama_models_dir}" // Allocate matching CPU/RAM to feed the GPU cpus = 2 @@ -58,13 +59,13 @@ profiles { report { enabled = true - file = "data/nextflow_logs/report.html" + file = "${params.outdir}/pipeline_info/report.html" overwrite = true } timeline { enabled = true - file = "data/nextflow_logs/timeline.html" + file = "${params.outdir}/pipeline_info/timeline.html" overwrite = true } From e5a3340cbaae03479fcc693aef2b1771cf31a152 Mon Sep 17 00:00:00 2001 From: Dmitrijs Lvovs Date: Wed, 5 Aug 2026 19:16:56 -0400 Subject: [PATCH 5/9] update resources --- main.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/main.nf b/main.nf index 34731d4..1d788c6 100644 --- a/main.nf +++ b/main.nf @@ -1,9 +1,8 @@ process INTERPRET { tag "${report.baseName}" label "process_gpu" - resourceLimits cpus: 4, memory: 24.GB, time: '1h' - //container 'ghcr.io/fertiglab/llmize:latest' - container 'ghcr.io/fertiglab/llmize:sha-f2ae922' + resourceLimits cpus: 2, memory: 8.GB, time: '1h' + container 'ghcr.io/fertiglab/llmize:latest' publishDir params.outdir, mode: 'copy' input: From 5abad5a6f2a3b14cc3ab5ef5a745237db1bc46de Mon Sep 17 00:00:00 2001 From: Dmitrijs Lvovs Date: Tue, 11 Aug 2026 10:23:59 -0400 Subject: [PATCH 6/9] implement review suggestions --- nextflow.config | 6 +++--- pipeline.py | 9 ++------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/nextflow.config b/nextflow.config index 7b8256f..891a885 100644 --- a/nextflow.config +++ b/nextflow.config @@ -38,7 +38,7 @@ profiles { process { executor = 'slurm' clusterOptions = { - account ? "-A ${params.slurm_account?.toString()?.trim()}" : '' + "-A ${params.slurm_account?.toString()?.trim()}" : '' } errorStrategy = 'retry' maxRetries = 3 @@ -50,8 +50,8 @@ profiles { // Allocate matching CPU/RAM to feed the GPU cpus = 2 - memory = '32 GB' - time = '4h' + memory = '8.GB' + time = '1h' } } } diff --git a/pipeline.py b/pipeline.py index 288591e..5c0193e 100644 --- a/pipeline.py +++ b/pipeline.py @@ -66,8 +66,8 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--output", "-o", - default=None, - help="Path to save the final interpreted report text. Defaults to data/_interpretation_.txt.", + default=".", + help="Path to save the final interpreted report text. Defaults to ./_interpretation_.txt.", ) parser.add_argument( "--num_ctx", @@ -122,11 +122,6 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Run environment preflight checks (Ollama, models, schema) and exit.", ) - parser.add_argument( - "--work-dir", - default=".", - help="Working directory for saving intermediate and final output files (default: current directory).", - ) return parser.parse_args() From e38ce6322f27eebde3a8aff26dd810c1343bffc2 Mon Sep 17 00:00:00 2001 From: Dmitrijs Lvovs Date: Thu, 13 Aug 2026 08:04:00 -0400 Subject: [PATCH 7/9] save to path or current dir --- main.nf | 4 ++-- nextflow.config | 2 +- pipeline.py | 12 ++++-------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/main.nf b/main.nf index 1d788c6..4888967 100644 --- a/main.nf +++ b/main.nf @@ -40,9 +40,9 @@ process INTERPRET { --input '${report}' \\ --model '${params.model}' \\ --num_ctx ${params.num_ctx} \\ - --work-dir . \\ ${think_flag} ${review_flag} ${whole_flag} ${synth_flag} \\ - ${prompt_flag} ${temp_flag} ${seed_flag} ${top_p_flag} ${top_k_flag} ${numpred_flag} \\ + ${prompt_flag} ${temp_flag} ${seed_flag} ${top_p_flag} \\ + ${top_k_flag} ${numpred_flag} \\ --output "${report.baseName}_interpretation_\${STAMP}.md" """ } diff --git a/nextflow.config b/nextflow.config index 891a885..c5548c9 100644 --- a/nextflow.config +++ b/nextflow.config @@ -38,7 +38,7 @@ profiles { process { executor = 'slurm' clusterOptions = { - "-A ${params.slurm_account?.toString()?.trim()}" : '' + "-A ${params.slurm_account?.toString()?.trim()}" } errorStrategy = 'retry' maxRetries = 3 diff --git a/pipeline.py b/pipeline.py index 5c0193e..d34b1f3 100644 --- a/pipeline.py +++ b/pipeline.py @@ -13,7 +13,6 @@ resolve_path, load_json, save_json, - DATA_DIR, extract_report_saved_raw_data, extract_focal_labels, annotate, @@ -66,7 +65,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--output", "-o", - default=".", + default=None, help="Path to save the final interpreted report text. Defaults to ./_interpretation_.txt.", ) parser.add_argument( @@ -152,7 +151,6 @@ def run_pipeline( annotated_filename: str | None, output_path: str | None, num_ctx: int, - work_dir: str = ".", whole_report: bool = False, synthesize_final: bool = True, think: bool = True, @@ -176,8 +174,8 @@ def run_pipeline( print(f"[pipeline] Reduced and annotated {len(report)} sections in memory.") if save_intermediates: - save_json(reduced, DATA_DIR, extracted_filename or default_output_name(input_path, prefix="extracted_")) - save_json(report, DATA_DIR, annotated_filename or "annotated_report.json") + save_json(reduced, ".", extracted_filename or default_output_name(".", prefix="extracted_")) + save_json(report, ".", annotated_filename or "annotated_report.json") mode = "whole report" if whole_report else "section-by-section" print(f"[pipeline] Calling Ollama model '{model}' ({mode})...") @@ -202,8 +200,7 @@ def run_pipeline( if output_path is None: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") stem = os.path.splitext(os.path.basename(input_path))[0] - output_filename = f"{stem}_interpretation_{timestamp}.md" - output_path = os.path.join(work_dir, output_filename) + output_path = f"{stem}_interpretation_{timestamp}.md" footer = build_run_footer( model=model, num_ctx=num_ctx, think=think, gen_options=gen_options, @@ -233,7 +230,6 @@ def main() -> None: annotated_filename=args.annotated_output, output_path=args.output, num_ctx=args.num_ctx, - work_dir=args.work_dir, whole_report=args.whole_report, synthesize_final=not args.no_synthesis, think=args.think, From f32a01aaf132f93b5d6263a9cbb5d86b9eb448f6 Mon Sep 17 00:00:00 2001 From: Dmitrijs Lvovs Date: Thu, 13 Aug 2026 09:59:13 -0400 Subject: [PATCH 8/9] update resource spec --- main.nf | 4 ++-- nextflow.config | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/main.nf b/main.nf index 4888967..0b5fcbd 100644 --- a/main.nf +++ b/main.nf @@ -1,8 +1,8 @@ process INTERPRET { tag "${report.baseName}" label "process_gpu" - resourceLimits cpus: 2, memory: 8.GB, time: '1h' - container 'ghcr.io/fertiglab/llmize:latest' + resourceLimits cpus: 2, memory: 48.GB, time: '1h' + container "${params.container}" publishDir params.outdir, mode: 'copy' input: diff --git a/nextflow.config b/nextflow.config index c5548c9..f1d002c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,7 +1,6 @@ params { input = null outdir = 'results' - model = 'gemma4' think = true review = false @@ -16,6 +15,7 @@ params { top_k = null num_predict = null ollama_models_dir = null + container = 'ghcr.io/fertiglab/llmize:latest' } @@ -45,12 +45,12 @@ profiles { withLabel: 'process_gpu' { // Allocate a gpu clusterOptions = "--gres=gpu:1 -A ${params.slurm_account?.toString()?.trim()}" - // Enable Nvidia in apptainer and mound ollama dir + // Enable Nvidia in apptainer and mount ollama dir containerOptions = "--nv -B ${params.ollama_models_dir}" // Allocate matching CPU/RAM to feed the GPU cpus = 2 - memory = '8.GB' + memory = '24.GB' time = '1h' } } From 83f213df4998978b29ab2cd1ad5510ac2fe64999 Mon Sep 17 00:00:00 2001 From: dimalvovs Date: Thu, 13 Aug 2026 10:42:32 -0400 Subject: [PATCH 9/9] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 418b13f..1c3eb40 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,9 @@ Optional features (e.g. ToolUniverse enrichment) only produce warnings, not fail The Nextflow module uses an Ollama model cache directory via `OLLAMA_MODELS`. -- In the `igs` and `igs_cpu` profiles, the default cache path is user-specific scratch: - `/usr/local/scratch/$USER/ollama/models` -- On the first run, if the model is missing, the workflow auto-pulls it. -- On later runs, the same model is reused from scratch and pull is skipped. +- By default, the workflow uses a task-local cache at `$PWD/ollama/models` (inside the Nextflow work directory). +- To reuse models across runs (recommended on clusters), pass `--ollama_models_dir /path/to/persistent/models` so the container can bind-mount that directory. +- On the first run with an empty cache, the workflow auto-pulls the model; subsequent runs reuse the cached model when using a persistent `--ollama_models_dir`. ### GPU (Slurm + Apptainer)