diff --git a/README.md b/README.md index b5d9cf6..1c3eb40 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,39 @@ python3 pipeline.py --check It prints a clear ✓/⚠/✗ report and exits non-zero if a required check fails. Optional features (e.g. ToolUniverse enrichment) only produce warnings, not failures. +## Nextflow execution + +The Nextflow module uses an Ollama model cache directory via `OLLAMA_MODELS`. + +- By default, the workflow uses a task-local cache at `$PWD/ollama/models` (inside the Nextflow work directory). +- To reuse models across runs (recommended on clusters), pass `--ollama_models_dir /path/to/persistent/models` so the container can bind-mount that directory. +- On the first run with an empty cache, the workflow auto-pulls the model; subsequent runs reuse the cached model when using a persistent `--ollama_models_dir`. + +### GPU (Slurm + Apptainer) + +```bash +nextflow run main.nf \ + -profile igs \ + --input data/multiqc_data.json \ + --slurm_account \ + -w /usr/local/scratch/$USER/work \ + -resume +``` + +### Override cache path (optional) + +Use this if your cluster requires a different location: + +```bash +nextflow run main.nf \ + -profile igs \ + --input data/multiqc_data.json \ + --slurm_account \ + --ollama_models_dir /path/to/persistent/models \ + -w /usr/local/scratch/$USER/work \ + -resume +``` + ## Continuous integration The GitHub Actions workflow (`.github/workflows/test.yml`) runs on every pull request diff --git a/main.nf b/main.nf index 3e5baac..0b5fcbd 100644 --- a/main.nf +++ b/main.nf @@ -1,6 +1,8 @@ process INTERPRET { tag "${report.baseName}" - container 'llmize:latest' + label "process_gpu" + resourceLimits cpus: 2, memory: 48.GB, time: '1h' + container "${params.container}" publishDir params.outdir, mode: 'copy' input: @@ -12,6 +14,8 @@ process INTERPRET { script: def home = workflow.containerEngine ? '/opt/llmize' : "${projectDir}" def boot = workflow.containerEngine ? "export LLMIZE_MODEL='${params.model}'\n bash ${home}/docker/boot_ollama.sh" : '' + def ollama_models_escaped = params.ollama_models_dir ? params.ollama_models_dir.toString().replace("'", "'\"'\"'") : null + def ollama_models_export = ollama_models_escaped ? "export OLLAMA_MODELS='${ollama_models_escaped}'" : 'export OLLAMA_MODELS="\$PWD/ollama/models"' def think_flag = "${params.think}".toBoolean() ? '--think' : '--no-think' def review_flag = "${params.review}".toBoolean() ? "--review --review-passes ${params.review_passes}" : '' def whole_flag = "${params.whole_report}".toBoolean() ? '--whole-report' : '' @@ -24,14 +28,21 @@ process INTERPRET { def top_k_flag = params.top_k != null ? "--top_k ${params.top_k}" : '' def numpred_flag = params.num_predict != null ? "--num_predict ${params.num_predict}" : '' """ + export HOME="\$PWD" + export XDG_CACHE_HOME="\$PWD/.cache" + ${ollama_models_export} + mkdir -p "\$OLLAMA_MODELS" "\$XDG_CACHE_HOME" + ${boot} + STAMP=\$(date +%Y%m%d_%H%M%S) python3 ${home}/pipeline.py \\ --input '${report}' \\ --model '${params.model}' \\ --num_ctx ${params.num_ctx} \\ ${think_flag} ${review_flag} ${whole_flag} ${synth_flag} \\ - ${prompt_flag} ${temp_flag} ${seed_flag} ${top_p_flag} ${top_k_flag} ${numpred_flag} \\ + ${prompt_flag} ${temp_flag} ${seed_flag} ${top_p_flag} \\ + ${top_k_flag} ${numpred_flag} \\ --output "${report.baseName}_interpretation_\${STAMP}.md" """ } diff --git a/nextflow.config b/nextflow.config index 994d09e..f1d002c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,7 +1,6 @@ params { input = null outdir = 'results' - model = 'gemma4' think = true review = false @@ -15,33 +14,58 @@ params { top_p = null top_k = null num_predict = null + ollama_models_dir = null + container = 'ghcr.io/fertiglab/llmize:latest' } -docker { - enabled = true -} profiles { + docker { + docker.enabled = true + } native { docker.enabled = false } + igs { + apptainer { + enabled = true + autoMounts = true + pullTimeout = '60m' + } + params { + slurm_account = null + } + process { + executor = 'slurm' + clusterOptions = { + "-A ${params.slurm_account?.toString()?.trim()}" + } + errorStrategy = 'retry' + maxRetries = 3 + withLabel: 'process_gpu' { + // Allocate a gpu + clusterOptions = "--gres=gpu:1 -A ${params.slurm_account?.toString()?.trim()}" + // Enable Nvidia in apptainer and mount ollama dir + containerOptions = "--nv -B ${params.ollama_models_dir}" + + // Allocate matching CPU/RAM to feed the GPU + cpus = 2 + memory = '24.GB' + time = '1h' + } + } + } } report { enabled = true - file = "data/nextflow_logs/report.html" + file = "${params.outdir}/pipeline_info/report.html" overwrite = true } timeline { enabled = true - file = "data/nextflow_logs/timeline.html" + file = "${params.outdir}/pipeline_info/timeline.html" overwrite = true } -process { - withName: INTERPRET { - container = 'llmize:latest' - containerOptions = "-v ${System.getProperty('user.home')}/.llmize-ollama:/root/.ollama" - } -} diff --git a/pipeline.py b/pipeline.py index 6495686..d34b1f3 100644 --- a/pipeline.py +++ b/pipeline.py @@ -13,7 +13,6 @@ resolve_path, load_json, save_json, - DATA_DIR, extract_report_saved_raw_data, extract_focal_labels, annotate, @@ -67,7 +66,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--output", "-o", default=None, - help="Path to save the final interpreted report text. Defaults to data/_interpretation_.txt.", + help="Path to save the final interpreted report text. Defaults to ./_interpretation_.txt.", ) parser.add_argument( "--num_ctx", @@ -175,8 +174,8 @@ def run_pipeline( print(f"[pipeline] Reduced and annotated {len(report)} sections in memory.") if save_intermediates: - save_json(reduced, DATA_DIR, extracted_filename or default_output_name(input_path, prefix="extracted_")) - save_json(report, DATA_DIR, annotated_filename or "annotated_report.json") + save_json(reduced, ".", extracted_filename or default_output_name(".", prefix="extracted_")) + save_json(report, ".", annotated_filename or "annotated_report.json") mode = "whole report" if whole_report else "section-by-section" print(f"[pipeline] Calling Ollama model '{model}' ({mode})...") @@ -201,8 +200,7 @@ def run_pipeline( if output_path is None: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") stem = os.path.splitext(os.path.basename(input_path))[0] - output_filename = f"{stem}_interpretation_{timestamp}.md" - output_path = os.path.join(DATA_DIR, output_filename) + output_path = f"{stem}_interpretation_{timestamp}.md" footer = build_run_footer( model=model, num_ctx=num_ctx, think=think, gen_options=gen_options,