Nextflow

README.md

@@ -53,6 +53,39 @@ python3 pipeline.py --check
 It prints a clear ✓/⚠/✗ report and exits non-zero if a required check fails.
 Optional features (e.g. ToolUniverse enrichment) only produce warnings, not failures.
 
+## Nextflow execution
+
+The Nextflow module uses an Ollama model cache directory via `OLLAMA_MODELS`.
+
+- By default, the workflow uses a task-local cache at `$PWD/ollama/models` (inside the Nextflow work directory).
+- To reuse models across runs (recommended on clusters), pass `--ollama_models_dir /path/to/persistent/models` so the container can bind-mount that directory.
+- On the first run with an empty cache, the workflow auto-pulls the model; subsequent runs reuse the cached model when using a persistent `--ollama_models_dir`.
+
+### GPU (Slurm + Apptainer)
+
+```bash
+nextflow run main.nf \
+   -profile igs \
+   --input data/multiqc_data.json \
+   --slurm_account <your-account> \
+   -w /usr/local/scratch/$USER/work \
+   -resume
+```
+
+### Override cache path (optional)
+
+Use this if your cluster requires a different location:
+
+```bash
+nextflow run main.nf \
+   -profile igs \
+   --input data/multiqc_data.json \
+   --slurm_account <your-account> \
+   --ollama_models_dir /path/to/persistent/models \
+   -w /usr/local/scratch/$USER/work \
+   -resume
+```
+
 ## Continuous integration
 
 The GitHub Actions workflow (`.github/workflows/test.yml`) runs on every pull request

main.nf

@@ -1,6 +1,8 @@
 process INTERPRET {
     tag "${report.baseName}"
-    container 'llmize:latest'
+    label "process_gpu"
+    resourceLimits cpus: 2, memory: 48.GB, time: '1h'
+    container "${params.container}"
     publishDir params.outdir, mode: 'copy'
 
     input:
@@ -12,6 +14,8 @@ process INTERPRET {
     script:
     def home = workflow.containerEngine ? '/opt/llmize' : "${projectDir}"
     def boot = workflow.containerEngine ? "export LLMIZE_MODEL='${params.model}'\n    bash ${home}/docker/boot_ollama.sh" : ''
+    def ollama_models_escaped = params.ollama_models_dir ? params.ollama_models_dir.toString().replace("'", "'\"'\"'") : null
+    def ollama_models_export = ollama_models_escaped ? "export OLLAMA_MODELS='${ollama_models_escaped}'" : 'export OLLAMA_MODELS="\$PWD/ollama/models"'
     def think_flag   = "${params.think}".toBoolean()        ? '--think' : '--no-think'
     def review_flag  = "${params.review}".toBoolean()       ? "--review --review-passes ${params.review_passes}" : ''
     def whole_flag   = "${params.whole_report}".toBoolean() ? '--whole-report' : ''
@@ -24,14 +28,21 @@ process INTERPRET {
     def top_k_flag   = params.top_k       != null ? "--top_k ${params.top_k}" : ''
     def numpred_flag = params.num_predict != null ? "--num_predict ${params.num_predict}" : ''
     """
+    export HOME="\$PWD"
+    export XDG_CACHE_HOME="\$PWD/.cache"
+    ${ollama_models_export}
+    mkdir -p "\$OLLAMA_MODELS" "\$XDG_CACHE_HOME"
+
     ${boot}
+
     STAMP=\$(date +%Y%m%d_%H%M%S)
     python3 ${home}/pipeline.py \\
         --input '${report}' \\
         --model '${params.model}' \\
         --num_ctx ${params.num_ctx} \\
         ${think_flag} ${review_flag} ${whole_flag} ${synth_flag} \\
-        ${prompt_flag} ${temp_flag} ${seed_flag} ${top_p_flag} ${top_k_flag} ${numpred_flag} \\
+        ${prompt_flag} ${temp_flag} ${seed_flag} ${top_p_flag} \\
+        ${top_k_flag} ${numpred_flag} \\
         --output "${report.baseName}_interpretation_\${STAMP}.md"
     """
 }

nextflow.config

@@ -1,7 +1,6 @@
 params {
     input        = null
     outdir       = 'results'
-
     model        = 'gemma4'
     think        = true
     review       = false
@@ -15,33 +14,58 @@ params {
     top_p        = null
     top_k        = null
     num_predict  = null
+    ollama_models_dir = null
+    container = 'ghcr.io/fertiglab/llmize:latest'
 }
 
-docker {
-    enabled = true
-}
 
 profiles {
+    docker {
+        docker.enabled = true
+    }
     native {
         docker.enabled = false
     }
+    igs {
+        apptainer {
+            enabled = true
+            autoMounts = true
+            pullTimeout = '60m'
+        }
+        params {
+            slurm_account = null
+        }
+        process {
+            executor = 'slurm'
+            clusterOptions = {
+                "-A ${params.slurm_account?.toString()?.trim()}"
+            }
+            errorStrategy =  'retry'
+            maxRetries = 3
+            withLabel: 'process_gpu' {
+                // Allocate a gpu
+                clusterOptions = "--gres=gpu:1 -A ${params.slurm_account?.toString()?.trim()}"
+                // Enable Nvidia in apptainer and mount ollama dir
+                containerOptions = "--nv -B ${params.ollama_models_dir}"
+
+            // Allocate matching CPU/RAM to feed the GPU
+            cpus = 2
+            memory = '24.GB'
+            time = '1h'
+            }
+        }
+    }
 }
 
 report {
     enabled   = true
-    file      = "data/nextflow_logs/report.html"
+    file      = "${params.outdir}/pipeline_info/report.html"
     overwrite = true
 }
 
 timeline {
     enabled   = true
-    file      = "data/nextflow_logs/timeline.html"
+    file      = "${params.outdir}/pipeline_info/timeline.html"
     overwrite = true
 }
 
-process {
-    withName: INTERPRET {
-        container = 'llmize:latest'
-        containerOptions = "-v ${System.getProperty('user.home')}/.llmize-ollama:/root/.ollama"
-    }
-}

pipeline.py

@@ -13,7 +13,6 @@
     resolve_path,
     load_json,
     save_json,
-    DATA_DIR,
     extract_report_saved_raw_data,
     extract_focal_labels,
     annotate,
@@ -67,7 +66,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--output", "-o",
         default=None,
-        help="Path to save the final interpreted report text. Defaults to data/<input_stem>_interpretation_<timestamp>.txt.",
+        help="Path to save the final interpreted report text. Defaults to ./<input_stem>_interpretation_<timestamp>.txt.",
     )
     parser.add_argument(
         "--num_ctx",
@@ -175,8 +174,8 @@ def run_pipeline(
     print(f"[pipeline] Reduced and annotated {len(report)} sections in memory.")
 
     if save_intermediates:
-        save_json(reduced, DATA_DIR, extracted_filename or default_output_name(input_path, prefix="extracted_"))
-        save_json(report, DATA_DIR, annotated_filename or "annotated_report.json")
+        save_json(reduced, ".", extracted_filename or default_output_name(".", prefix="extracted_"))
+        save_json(report, ".", annotated_filename or "annotated_report.json")
 
     mode = "whole report" if whole_report else "section-by-section"
     print(f"[pipeline] Calling Ollama model '{model}' ({mode})...")
@@ -201,8 +200,7 @@ def run_pipeline(
     if output_path is None:
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         stem = os.path.splitext(os.path.basename(input_path))[0]
-        output_filename = f"{stem}_interpretation_{timestamp}.md"
-        output_path = os.path.join(DATA_DIR, output_filename)
+        output_path = f"{stem}_interpretation_{timestamp}.md"
 
     footer = build_run_footer(
         model=model, num_ctx=num_ctx, think=think, gen_options=gen_options,