From f2540e08e6a8d8a0cee0541298fbbea6977d3abb Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:32:19 +0100 Subject: [PATCH] New modelblocks and unit test directories and tasks --- CHANGELOG | 7 ++ README.md | 14 ++-- copier.yaml | 13 ++-- template/.github/workflows/pr-ci.yml | 16 +++-- template/README.md.jinja | 7 +- template/config/README.md | 4 +- template/figures/.gitkeep | 1 - template/pixi.toml.jinja | 11 ++- template/tests/conftest.py | 12 ---- template/tests/modelblocks/conftest.py | 20 ++++++ .../convention_test.py} | 35 ++-------- .../integration/test_config.yaml.jinja | 0 .../{% if true %}Snakefile{% endif %}.jinja | 2 +- .../modelblocks/module_integration_test.py | 26 +++++++ template/tests/unit/conftest.py | 12 ++++ template/workflow/scripts/dummy_script.py | 24 +++++-- tests/template_test.py | 68 ++++++++++++++++++- 17 files changed, 201 insertions(+), 71 deletions(-) delete mode 100644 template/figures/.gitkeep delete mode 100644 template/tests/conftest.py create mode 100644 template/tests/modelblocks/conftest.py rename template/tests/{integration_test.py => modelblocks/convention_test.py} (65%) rename template/tests/{ => modelblocks}/integration/test_config.yaml.jinja (100%) rename template/tests/{ => modelblocks}/integration/{% if true %}Snakefile{% endif %}.jinja (97%) create mode 100644 template/tests/modelblocks/module_integration_test.py create mode 100644 template/tests/unit/conftest.py diff --git a/CHANGELOG b/CHANGELOG index c3c27b3..dabc83b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,12 @@ # Changelog +## 0.2.0 + +- Change: `tests/` has been separated into `tests/modelblocks` and `tests/unit` (convention and unit testing, respectively). + - `tests/integration_test.py` was split into `convention_test.py` (interfacing checks) and `module_integration_test.py` (module export checks). +- Add: new `unit-test` pixi environment, resolved in tandem with the `module` environment while adding `pytest` to it. +- Add: new `pixi run test-unit` task for unit testing script code. + ## 0.1.0 ### Added diff --git a/README.md b/README.md index 59c27c2..f67bc0a 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ A template for modular data workflows built with [`Snakemake`](https://snakemake > > - **Modules do not work like regular `Snakemake` workflows** > - They must be tested externally using the `module:` command in `Snakemake`, passing user resources, and requesting specific results. -Check the pre-made example in `tests/integration` for details. +Check the pre-made example in `tests/modelblocks/integration` for details. > - Internal access (e.g., calling the `all:` rule) is discouraged, as the module may not have the necessary `resources/` to execute properly. > - **Please be sure to maintain the following files to ensure Modelblocks compatibility** > - `INTERFACE.yaml`: a simple description of the module's input/output structure. @@ -82,7 +82,7 @@ This template uses [`pixi`](https://pixi.sh/) as its package manager. Once insta 6. Extra: run the auto-generated example module! ```shell - cd tests/integration # go to the integration test... + cd tests/modelblocks/integration # go to the integration test... pixi run snakemake --use-conda # run it! ``` @@ -93,10 +93,16 @@ This template uses [`pixi`](https://pixi.sh/) as its package manager. Once insta Export `` to `conda`-compatible dependency files, saved in `workflow/envs`, allowing `Snakemake` to use them during rule execution. This will generate both an `.yaml` file and platform-specific pin files for Windows, Linux and macOS (e.g., `.win-64.pin.txt`). -### `pixi run test-integration` +### `pixi run test-modelblocks` Run a minimal set of standardised tests to ensure your module complies with Modelblock requirements. -These are executed by Github's CI during pull requests. +These tests are executed by Github's CI during pull requests. + + +### `pixi run test-unit` + +Run unit tests on workflow python code in `workflow/scripts/`, using a separate `unit-test` environment. +These tests are executed by Github's CI during pull requests. ## Contributors ✨ diff --git a/copier.yaml b/copier.yaml index c841a71..ba8d6fa 100644 --- a/copier.yaml +++ b/copier.yaml @@ -17,10 +17,8 @@ _exclude: - "{% if _copier_operation == 'update' -%}figures{% endif %}" - "{% if _copier_operation == 'update' -%}results{% endif %}" - "{% if _copier_operation == 'update' -%}config/**{% endif %}" - - "{% if _copier_operation == 'update' -%}tests/**{% endif %}" # Special cases within the above that should not be excluded. - "{% if _copier_operation == 'update' -%}!config/README.md{% endif %}" - - "{% if _copier_operation == 'update' -%}!tests/integration_test.py{% endif %}" - "{% if _copier_operation == 'update' -%}!workflow/profiles/default/config.yaml{% endif %}" _skip_if_exists: # Respect user changes, except for a few standard files. @@ -28,14 +26,21 @@ _skip_if_exists: - resources/** - figures/** - results/** - - tests/** + - tests/unit/** - workflow/** # Special cases within the above that should not be skipped. - "!config/README.md" - - "!tests/integration_test.py" - "!workflow/profiles/default/config.yaml" _templates_suffix: .jinja _answers_file: .copier-answers.yml +_message_after_update: | + ⚠️⚠️ IMPORTANT: Integration test layout changed ⚠️⚠️ + + Modelblocks integration and compatibility tests now live in tests/modelblocks/. + If your project still uses tests/integration/, merge its Snakefile and supporting files into tests/modelblocks/integration/. + Adjust relative paths as needed. + + Run pixi run test-modelblocks to validate a successful migration. # === copier user provided === module_code_name: diff --git a/template/.github/workflows/pr-ci.yml b/template/.github/workflows/pr-ci.yml index 18c5f00..9a1c841 100644 --- a/template/.github/workflows/pr-ci.yml +++ b/template/.github/workflows/pr-ci.yml @@ -17,24 +17,28 @@ jobs: - uses: actions/checkout@v7 - name: Setup pixi uses: prefix-dev/setup-pixi@v0.9.6 - - name: Run integration tests - id: tests - run: pixi run test-integration + - name: Run standard Modelblocks tests + id: test_modelblocks + run: pixi run test-modelblocks + continue-on-error: true + - name: Run unit tests + id: test_unit + run: pixi run test-unit continue-on-error: true - name: Run snakemake linting id: linting run: | pixi run snakemake --lint --snakefile workflow/Snakefile - pixi run snakemake --lint --snakefile tests/integration/Snakefile + pixi run snakemake --lint --snakefile tests/modelblocks/integration/Snakefile continue-on-error: true - name: Save integration logs if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: integration-test-logs-${{ matrix.os }} - path: tests/integration/resources/module/logs + path: tests/modelblocks/integration/resources/module/logs if-no-files-found: ignore retention-days: 30 - name: Fail if integration or linting failed - if: ${{ steps.tests.outcome == 'failure' || steps.linting.outcome == 'failure' }} + if: ${{ steps.test_modelblocks.outcome == 'failure' || steps.test_unit.outcome == 'failure' || steps.linting.outcome == 'failure' }} run: exit 1 diff --git a/template/README.md.jinja b/template/README.md.jinja index b21af00..91594e0 100644 --- a/template/README.md.jinja +++ b/template/README.md.jinja @@ -14,7 +14,7 @@ This is a modular `snakemake` workflow created as part of the [Modelblocks project](https://www.modelblocks.org/). It can be imported directly into any `snakemake` workflow. For more information, please consult the Modelblocks [documentation](https://modelblocks.readthedocs.io/en/latest/), -the [integration example](./tests/integration/Snakefile), +the [integration example](./tests/modelblocks/integration/Snakefile), and the `snakemake` [documentation](https://snakemake.readthedocs.io/en/stable/snakefiles/modularization.html). ## Overview @@ -65,14 +65,15 @@ pixi run export-snakemake-env module For testing, simply run: ```shell -pixi run test-integration +pixi run test-modelblocks # Modelblocks compatibility +pixi run test-unit # Script unit tests ``` To test a minimal example of a workflow using this module: ```shell pixi shell # activate this project's environment -cd tests/integration/ # navigate to the integration example +cd tests/modelblocks/integration/ # navigate to the integration example snakemake --use-conda --cores 2 # run the workflow! ``` diff --git a/template/config/README.md b/template/config/README.md index 720820c..ae7b291 100644 --- a/template/config/README.md +++ b/template/config/README.md @@ -1,8 +1,10 @@ +# Configuration + We recommend consulting the following before using this module: - `config/config.yaml`: a generic example configuration of this module. - `workflow/internal/config.schema.yaml`: a schematic overview of all the configuration options of this module. - `INTERFACE.yaml`: lists module input and output files, and their default locations. -- `tests/integration/Snakefile`: an example of how to call this module from another workflow. +- `tests/modelblocks/integration/Snakefile`: an example of how to call this module from another workflow. This data module is part of the [Modelblocks](https://www.modelblocks.org/) project. Please consult the [Modelblocks documentation](https://modelblocks.readthedocs.io/) for more details. diff --git a/template/figures/.gitkeep b/template/figures/.gitkeep deleted file mode 100644 index dcafcbe..0000000 --- a/template/figures/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# For module documentation figures diff --git a/template/pixi.toml.jinja b/template/pixi.toml.jinja index a2e14a3..625de32 100644 --- a/template/pixi.toml.jinja +++ b/template/pixi.toml.jinja @@ -25,11 +25,18 @@ pytz = ">=2026.1.post1" [feature.module.dependencies] curl = ">=8.9.1" +[feature.testing.dependencies] +pytest = ">=8.3.5" + +[feature.testing.tasks] +test-unit = {cmd = "pytest tests/unit/", default-environment = "unit-test"} + [environments] -module = { features = ["module"], no-default-feature = true } +module = { features = ["module"], no-default-feature = true, solve-group = "module" } +unit-test = { features = ["module", "testing"], no-default-feature = true, solve-group = "module" } [tasks] -test-integration = {cmd = "pytest tests/integration_test.py"} +test-modelblocks = {cmd = "pytest tests/modelblocks/", default-environment = "default"} {% raw %} [tasks.export-snakemake-env] diff --git a/template/tests/conftest.py b/template/tests/conftest.py deleted file mode 100644 index f5a5699..0000000 --- a/template/tests/conftest.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Shared pytest fixtures.""" - -from pathlib import Path - -import pytest - - -@pytest.fixture(scope="module") -def module_path(): - """Parent directory of the project.""" - # If your module needs files in resources/user/, place automated downloads here. - return Path(__file__).parent.parent diff --git a/template/tests/modelblocks/conftest.py b/template/tests/modelblocks/conftest.py new file mode 100644 index 0000000..0ddfe8b --- /dev/null +++ b/template/tests/modelblocks/conftest.py @@ -0,0 +1,20 @@ +"""Shared pytest fixtures.""" + +import tomllib +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def module_path(): + """Parent directory of the project.""" + # If your module needs files in resources/user/, place automated downloads here. + return Path(__file__).parents[2] + + +@pytest.fixture(scope="module") +def pixi_platforms(module_path) -> list[str]: + """Pixi platforms defined for this project.""" + with (module_path / "pixi.toml").open("rb") as pixi_config: + return tomllib.load(pixi_config)["workspace"]["platforms"] diff --git a/template/tests/integration_test.py b/template/tests/modelblocks/convention_test.py similarity index 65% rename from template/tests/integration_test.py rename to template/tests/modelblocks/convention_test.py index c56b086..c66e3f9 100644 --- a/template/tests/integration_test.py +++ b/template/tests/modelblocks/convention_test.py @@ -1,24 +1,17 @@ -"""Set of standard Modelblocks tests. +"""Test Modelblocks convention compliance. +!!!!! DO NOT ERASE OR MODIFY !!!!! PLEASE ENSURE THIS SET OF MINIMAL TESTS WORKS BEFORE PUBLISHING YOUR MODULE. Contents may be updated in future template updates. """ import subprocess -import tomllib from pathlib import Path import pytest from clio_tools.data_module import ModuleInterface -@pytest.fixture(scope="module") -def pixi_platforms(module_path) -> list[str]: - """Pixi platforms defined for this project.""" - with (module_path / "pixi.toml").open("rb") as pixi_config: - return tomllib.load(pixi_config)["workspace"]["platforms"] - - def test_snakemake_environments(module_path, pixi_platforms, tmp_path): """All Snakemake environment files should be based on pixi counterparts.""" env_dir = module_path / "workflow/envs" @@ -57,28 +50,14 @@ def test_interface_file(module_path): "LICENSE", "README.md", "config/config.yaml", + "config/README.md", + "figures/module.png", + "tests/modelblocks/integration/Snakefile", "workflow/internal/config.schema.yaml", - "tests/integration/Snakefile", + ".github/workflows/check-version.yml", + ".github/workflows/pr-ci.yml", ], ) def test_standard_file_existance(module_path, file): """Check that a minimal set of files used for documentation are present.""" assert Path(module_path / file).exists() - - -def test_snakemake_all_failure(module_path): - """The snakemake 'all' rule should return an error by default.""" - process = subprocess.run( - "snakemake --cores 1", shell=True, cwd=module_path, capture_output=True - ) - assert "INVALID" in str(process.stderr) - - -def test_snakemake_integration_testing(module_path): - """Run a light-weight test simulating someone using this module.""" - assert subprocess.run( - "snakemake --use-conda --cores 1", - shell=True, - check=True, - cwd=module_path / "tests/integration", - ) diff --git a/template/tests/integration/test_config.yaml.jinja b/template/tests/modelblocks/integration/test_config.yaml.jinja similarity index 100% rename from template/tests/integration/test_config.yaml.jinja rename to template/tests/modelblocks/integration/test_config.yaml.jinja diff --git a/template/tests/integration/{% if true %}Snakefile{% endif %}.jinja b/template/tests/modelblocks/integration/{% if true %}Snakefile{% endif %}.jinja similarity index 97% rename from template/tests/integration/{% if true %}Snakefile{% endif %}.jinja rename to template/tests/modelblocks/integration/{% if true %}Snakefile{% endif %}.jinja index 617c374..7523285 100644 --- a/template/tests/integration/{% if true %}Snakefile{% endif %}.jinja +++ b/template/tests/modelblocks/integration/{% if true %}Snakefile{% endif %}.jinja @@ -22,7 +22,7 @@ rule create_external_input: # `config`: specifies the module configuration. # `pathvars:` helps you re-wire where the module places files. module {{module_code_name}}: - snakefile: "../../workflow/Snakefile" + snakefile: "../../../workflow/Snakefile" config: config["{{module_code_name}}"] pathvars: # Redirect specific user resources (inputs) diff --git a/template/tests/modelblocks/module_integration_test.py b/template/tests/modelblocks/module_integration_test.py new file mode 100644 index 0000000..8dacbf0 --- /dev/null +++ b/template/tests/modelblocks/module_integration_test.py @@ -0,0 +1,26 @@ +"""Test importing the module through Snakemake. + +!!!!! DO NOT ERASE OR MODIFY !!!!! +PLEASE ENSURE THIS SET OF MINIMAL TESTS WORKS BEFORE PUBLISHING YOUR MODULE. +Contents may be updated in future template updates. +""" + +import subprocess + + +def test_snakemake_all_failure(module_path): + """The snakemake 'all' rule should return an error by default.""" + process = subprocess.run( + "snakemake --cores 1", shell=True, cwd=module_path, capture_output=True + ) + assert "INVALID" in str(process.stderr) + + +def test_snakemake_integration_testing(module_path): + """Run a light-weight test simulating someone using this module.""" + assert subprocess.run( + "snakemake --use-conda --cores 1 --forceall", + shell=True, + check=True, + cwd=module_path / "tests/modelblocks/integration", + ) diff --git a/template/tests/unit/conftest.py b/template/tests/unit/conftest.py new file mode 100644 index 0000000..8648724 --- /dev/null +++ b/template/tests/unit/conftest.py @@ -0,0 +1,12 @@ +"""Unit tests for module script code. + +IMPORTANT: must be executed in the "unit-test" environment. +""" + +import sys +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parent.parent.parent +WORKFLOW_SCRIPTS = MODULE_PATH / "workflow" / "scripts" + +sys.path.insert(0, str(WORKFLOW_SCRIPTS)) diff --git a/template/workflow/scripts/dummy_script.py b/template/workflow/scripts/dummy_script.py index 4ceba9b..da8f815 100644 --- a/template/workflow/scripts/dummy_script.py +++ b/template/workflow/scripts/dummy_script.py @@ -9,12 +9,24 @@ if TYPE_CHECKING: snakemake: Any -sys.stderr = open(snakemake.log[0], "w") -config = snakemake.params.config_text -readme = Path(snakemake.input.readme).read_text() -user = Path(snakemake.input.user_message).read_text() -output_text = "\n\n".join([readme, user, config]) +def dummy_uppercase(text: str) -> str: + """Turns text to uppercase.""" + return text.upper() -Path(snakemake.output.combined_text).write_text(output_text) + +def main(): + """Main Snakemake process.""" + config = snakemake.params.config_text + readme = Path(snakemake.input.readme).read_text() + user = Path(snakemake.input.user_message).read_text() + + output_text = dummy_uppercase("\n\n".join([readme, user, config])) + + Path(snakemake.output.combined_text).write_text(output_text) + + +if __name__ == "__main__": + sys.stderr = open(snakemake.log[0], "w", buffering=1) + main() diff --git a/tests/template_test.py b/tests/template_test.py index 18cb4f9..022072c 100644 --- a/tests/template_test.py +++ b/tests/template_test.py @@ -1,5 +1,6 @@ """Test a simplified template use-case.""" +import json import subprocess from copy import deepcopy from pathlib import Path @@ -119,11 +120,72 @@ def pixi_built(self, template_project): ) return template_project - def test_task_test_integration(self, pixi_built): - """The template's tests should pass by default.""" + @pytest.mark.parametrize( + ("task", "environment"), + [ + ("test-modelblocks", "default"), + ("test-unit", "unit-test"), + ("export-snakemake-env", "default"), + ], + ids=["test-modelblocks", "test-unit", "export-snakemake-env"], + ) + def test_task_environment(self, template_project, task, environment): + """Tasks should be available in their intended environment.""" + process = subprocess.run( + [ + "pixi", + "task", + "list", + "--manifest-path", + str(template_project / "pixi.toml"), + "--json", + ], + cwd=template_project, + capture_output=True, + text=True, + check=True, + ) + task_environments = set() + for environment_info in json.loads(process.stdout): + tasks = environment_info["tasks"] + [ + task_info + for feature in environment_info["features"] + for task_info in feature["tasks"] + ] + if any(task_info["name"] == task for task_info in tasks): + task_environments.add(environment_info["environment"]) + assert task_environments == {environment} + + def test_task_test_modelblocks(self, pixi_built): + """The template's Modelblock tests should pass by default.""" assert subprocess.run( - "pixi run test-integration", shell=True, check=True, cwd=pixi_built + "pixi run test-modelblocks", shell=True, check=True, cwd=pixi_built + ) + + def test_task_test_unit_default_failure(self, pixi_built): + """Unit testing should require tests and fail by default in CI.""" + process = subprocess.run("pixi run test-unit", shell=True, cwd=pixi_built) + assert process.returncode == pytest.ExitCode.NO_TESTS_COLLECTED + + def test_task_test_unit_with_tests(self, pixi_built): + """Unit testing should succeed when tests are included by users.""" + unit_test = pixi_built / "tests/unit/dummy_script_test.py" + unit_test.write_text( + '"""Example of script function validation."""\n' + "\n" + "import dummy_script\n" + "\n" + "\n" + "def test_dummy_uppercase():\n" + ' """Test the dummy uppercase function."""\n' + ' assert dummy_script.dummy_uppercase("modelblocks") == "MODELBLOCKS"\n' ) + try: + assert subprocess.run( + "pixi run test-unit", shell=True, check=True, cwd=pixi_built + ) + finally: + unit_test.unlink() def test_task_export_snakemake_env(self, pixi_built): """The template's export task should work as expected."""