diff --git a/CLAUDE.md b/CLAUDE.md index be2531cf..e53fd7d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,16 +1,18 @@ # Stackinator -Stackinator is a Python CLI tool that generates build configurations for scientific software stacks on HPE Cray EX (Alps) systems. It acts like `cmake`/`configure`: given a **recipe** and a **cluster configuration**, it produces a build directory with Makefiles and Spack YAML files. The actual build is then performed by `make`. +Stackinator is a Python CLI tool that generates build configurations for scientific software stacks on HPE Cray EX (Alps) systems. It acts like `cmake`/`configure`: given a **recipe** and a **cluster configuration**, it produces a build directory with a Makefile and a single `spack.yaml`. The actual build is then performed by `make`. + +**Stackinator v7 (this branch) supports only version 3 recipes (Spack 1.2+).** For version 2 recipes use the `releases/v6` branch. ## Two-Phase Workflow ``` -stack-config -b BUILD -r RECIPE -s SYSTEM [--mirror MIRRORS] [-m MOUNT] # [-c CACHE] is legacy - → generates BUILD/ directory with Makefiles + spack.yaml files +stack-config -b BUILD -r RECIPE -s SYSTEM [-c CACHE] [-m MOUNT] + → generates BUILD/ directory with Makefile + spack.yaml cd BUILD env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make store.squashfs -j64 - → clones Spack, concretises, builds, creates store.squashfs + → clones Spack, concretises all spec groups, builds, creates store.squashfs ``` The `store.squashfs` SquashFS image is the final artifact, intended to be mounted at the recipe's `store` path (default `/user-environment`) as a uenv image. @@ -22,29 +24,26 @@ stackinator/ # Python package main.py # CLI entry point (stack-config) recipe.py # Recipe class: parses and validates all recipe YAML builder.py # Builder class: writes all files to the build path - mirror.py # Mirrors class: validates the --mirror mirrors.yaml, emits spack mirror/cache config + cache.py # Build cache configuration helpers spack_util.py # Tiny helper: checks if a path is a spack package repo schema.py # JSON schema validators with default-injection schema/ # JSON schemas for each YAML file type config.json compilers.json environments.json - mirror.json # mirrors.yaml schema (buildcache/bootstrap/sourcemirror/sourcecache/concretizer) - cache.json # legacy -c/--cache cache.yaml schema + cache.json modules.json templates/ # Jinja2 templates for all generated files - Makefile # Top-level build orchestration - Makefile.compilers # Compiler build steps - Makefile.environments # Environment build + view generation steps + Makefile # Top-level build orchestration (single concretize + install) Makefile.generate-config # Generates upstream spack config for the uenv Make.user # Build path / store / sandbox variable definitions repos.yaml # Generated spack repos.yaml + spack.yaml # Unified spack.yaml with spec groups for all compilers + envs stack-debug.sh # Debug helper script - compilers.*.spack.yaml # Per-compiler spack.yaml configs - environments.spack.yaml # Environment spack.yaml config etc/ - Make.inc # Shared make rules (concretize, depfile, compiler_bin_dirs) + Make.inc # Shared make rules bwrap-mutable-root.sh # Bubblewrap sandbox wrapper + compiler-config.py # Spack Python script: generates packages.yaml with compiler externals envvars.py # CLI tool: generates env.json for views and uenv metadata docs/ # MkDocs documentation source unittests/ # pytest test suite @@ -61,10 +60,10 @@ A recipe is a directory containing YAML files: ```yaml name: prgenv-gnu store: /user-environment # mount point; default /user-environment -version: 2 # must be 2 for Spack 1.0 (Stackinator 6+) +version: 3 # must be 3; v1/v2 require Stackinator v6 spack: repo: https://github.com/spack/spack.git - commit: releases/v1.0 # branch, tag, or SHA; null = default branch + commit: releases/v1.2 # branch, tag, or SHA; null = default branch packages: repo: https://github.com/spack/spack-packages.git commit: develop @@ -72,7 +71,6 @@ description: "optional text" default-view: develop # optional: view loaded when no view is specified ``` -- `version: 1` (the default) targets Spack v0.23 and is only supported by Stackinator v5 (`releases/v5` branch). **Current `main` requires `version: 2`.** - `store` can be overridden at configure time with `-m/--mount`. ### `compilers.yaml` (required) @@ -89,7 +87,7 @@ intel-oneapi-compilers: # optional version: "2024.1" ``` -Build order: `gcc` is built first (using system compiler), then `nvhpc`/`llvm`/`llvm-amdgpu`/`intel-oneapi-compilers` are built using the gcc toolchain. Stackinator appends opinionated variants (e.g. `gcc@13 +bootstrap`, `nvhpc@25.1 ~mpi~blas~lapack`, `llvm@16 +clang ~gold`). +Build order: `gcc` is built first (using system compiler), then `nvhpc`/`llvm`/`llvm-amdgpu`/`intel-oneapi-compilers` are built using the gcc toolchain. Stackinator appends opinionated variants (e.g. `gcc@13 +bootstrap`, `nvhpc@25.1 ~mpi~blas~lapack`, `llvm@16 +clang ~gold`). Each compiler becomes a separate spec group in the unified `spack.yaml`. ### `environments.yaml` (required) ```yaml @@ -99,7 +97,7 @@ my-env: - cmake - hdf5+mpi network: # optional; null = no MPI - mpi: cray-mpich # full spack spec for MPI (cray-mpich or openmpi) + mpi: cray-mpich # MPI implementation name (must match network.yaml key) specs: ['libfabric@1.22'] # optional; overrides network.yaml defaults unify: true # concretizer: true | false | when_possible (default true) duplicates: @@ -109,9 +107,6 @@ my-env: - +cuda - cuda_arch=80 prefer: null # packages:all:prefer; auto-set if null - packages: # external packages to discover via `spack external find` - - perl - - git views: # optional filesystem views default: null # view name → view config (null = defaults) no-python: @@ -135,6 +130,7 @@ my-env: - Spec matrices are not supported. - Only one MPI per environment; create separate environments for multiple MPIs. - The `prefer` field is auto-generated if `null`: it nudges Spack to use the first compiler for all packages. +- The `packages` field (list of package names for `spack external find`) is accepted by the schema but **ignored in v3 recipes** — a warning is emitted. Add external packages to `packages.yaml` instead. #### Environment variable special syntax - `${@VAR@}` — deferred expansion: expands `VAR` at uenv load time (e.g. `${@HOME@}`) @@ -177,8 +173,6 @@ cluster-config/ repos.yaml # optional; list of relative paths to site-wide spack repos ``` -Mirror/cache config is **not** part of the cluster configuration — it is supplied separately with `--mirror` (a `mirrors.yaml` here is rejected). See [Mirrors and Build Caches](#mirrors-and-build-caches). - `network.yaml` structure: ```yaml mpi: @@ -191,38 +185,26 @@ packages: # standard spack packages.yaml content cray-mpich: ... ``` -Package precedence (recipe.py merges these): recipe `packages.yaml` > `network.yaml` packages > `packages.yaml` (minus gcc). The `gcc` entry from `packages.yaml` is isolated and used only for the gcc compiler build step. +Package precedence (`recipe.py` merges these): recipe `packages.yaml` > `network.yaml` packages > cluster `packages.yaml`. All packages (including gcc externals) go into a single global `packages.yaml` that is included by the unified `spack.yaml`. ## Build Directory Structure (output) ``` BUILD/ - Makefile # top-level orchestration + Makefile # top-level orchestration (single concretize + install) Make.user # variables: BUILD_ROOT, STORE, SANDBOX, etc. Make.inc # shared make rules (copied from etc/) bwrap-mutable-root.sh # sandbox wrapper (copied from etc/) envvars.py # view/meta generator (copied from etc/) + compiler-config.py # compiler external generator (copied from etc/) + spack.yaml # unified spack.yaml with all spec groups + packages.yaml # merged system + network + recipe packages + config.yaml # spack install tree location spack/ # cloned Spack repository spack-packages/ # cloned spack-packages repository - config/ # global spack configuration scope - packages.yaml + config/ # SPACK_SYSTEM_CONFIG_PATH scope repos.yaml - mirrors.yaml # if --mirror provided: buildcache/sourcemirror entries - config.yaml # if --mirror provided with a sourcecache (config:source_cache) - concretizer.yaml # if --mirror provided with a concretizer cache (concretizer:concretization_cache), spack >= 1.1 - bootstrap.yaml # if --mirror provided with a bootstrap mirror - key_store/ # if --mirror provided with gpg keys (decoded *.gpg) - compilers/ - Makefile - gcc/ - spack.yaml - packages.yaml # generated by spack external find - nvhpc/ # if nvhpc in recipe - spack.yaml - environments/ - Makefile - my-env/ - spack.yaml + mirrors.yaml # only if --cache provided generate-config/ # generates the upstream spack config for the final image Makefile modules/ # only if modules.yaml in recipe @@ -248,34 +230,32 @@ BUILD/ ### `Recipe` class (`recipe.py`) Parses and validates all recipe inputs in `__init__`. Key responsibilities: - Validates each YAML file against its JSON schema (with default injection) -- Merges packages from cluster config, network.yaml, and recipe +- Merges packages from cluster config, network.yaml, and recipe into a single dict - Generates full compiler specs (e.g. `gcc@13 +bootstrap`) from `compilers.yaml` - Processes environments: resolves MPI specs from `network.yaml` templates, sets default `prefer` constraints, builds view metadata -- Provides `compiler_files` and `environment_files` properties (Jinja-rendered Makefiles and spack.yaml files) +- Provides `spack_yaml` property (Jinja-rendered unified spack.yaml with spec groups) +- Provides `compiler_names` property (list of compiler package names for `compiler-config.py`) ### `Builder` class (`builder.py`) Writes all files to the build path. Key responsibilities: - Creates directory structure - Clones Spack and spack-packages repositories - Merges and writes the consolidated `alps` spack package repo -- Renders all Jinja templates into build path files +- Writes the unified `spack.yaml`, `packages.yaml`, and `config.yaml` to `BUILD_ROOT` +- Renders Makefile, Make.user, generate-config/Makefile from Jinja2 templates +- Copies `Make.inc`, `bwrap-mutable-root.sh`, `envvars.py`, `compiler-config.py` from `etc/` - Writes metadata JSON files -### `Mirrors` class (`mirror.py`) -A clean exemplar of the "recipe validates & renders, builder just prints" pattern. Constructed by `Recipe` from the `--mirror` file path; does ALL mirror input processing eagerly in `__init__` (loads + schema-validates `mirrors.yaml`, validates mirror urls, decodes/validates gpg keys to in-memory bytes, checks cache paths are absolute and expands env vars). Then presents pure static artifacts: -- typed members: `buildcache`, `bootstrap`, `source_mirrors`, `source_cache`, `concretizer_cache` -- `config_files(config_root) -> {abs_path: bytes}` — the `mirrors.yaml`, `config.yaml`, `concretizer.yaml`, `bootstrap.yaml`, and gpg key files the builder writes verbatim -- `gpg_key_paths(config_root)` and the `build_cache_mirror` / `push_to_build_cache` properties (the latter is `None` for a keyless, read-only build cache) - -Mirror/cache config is supplied ONLY via `--mirror`; a `mirrors.yaml` found in the system config dir is rejected with an error (it was never a system-config artifact). Relative gpg-key paths resolve against the `--mirror` file's own directory. - ### `schema.py` -JSON schema validation using `jsonschema`. The `validator()` function extends the validator to auto-inject `default` values from schemas into parsed instances, so downstream code can rely on optional fields always being present. +JSON schema validation using `jsonschema`. The `validator()` function extends the validator to auto-inject `default` values from schemas into parsed instances, so downstream code can rely on optional fields always being present. `check_config_version` enforces version 3 and gives a clear error for v1/v2 recipes pointing to the `releases/v6` branch. + +### `etc/compiler-config.py` +Replaces `spack compiler find`. Run as `spack -e BUILD_ROOT python compiler-config.py OUTPUT_YAML COMPILER...`. Uses `spack.store.STORE.db.query()` to find installed compiler packages, walks the install prefix to locate binaries, and writes (or merges into) a `packages.yaml` with `extra_attributes.compilers` entries. This is how compilers become available to downstream Spack users without a `compilers.yaml`. ### `etc/envvars.py` A standalone CLI tool (copied into the build directory) with two subcommands: -- `envvars.py view [--compilers] [--prefix_paths]`: reads a Spack-generated `activate.sh`, parses env vars, adds compiler symlinks and prefix paths, writes `env.json` for the view -- `envvars.py uenv [--modules] [--spack]`: merges view `env.json` files with recipe `env_vars` config, writes the final `meta/env.json` +- `envvars.py view [--compilers=FILE] [--prefix_paths=STR]`: reads a Spack-generated `activate.sh`, parses env vars, adds compiler symlinks and prefix paths, writes `env.json` for the view +- `envvars.py uenv [--modules] [--spack=...]`: merges view `env.json` files with recipe `env_vars` config, writes the final `meta/env.json` The `EnvVarSet` class in `envvars.py` is also imported by `recipe.py` for processing `env_vars` at configure time. @@ -283,39 +263,49 @@ The `EnvVarSet` class in `envvars.py` is also imported by `recipe.py` for proces The top-level `Makefile` orchestrates in order: 1. `spack-setup` — sanity check, bootstrap concretizer -2. `pre-install` — run `pre-install-hook` if provided -3. `mirror-setup` — trust build-cache/mirror gpg keys (`cache-force` force-pushes built packages) -4. `compilers` — build gcc, then nvhpc/llvm/etc. (parallel within each stage) -5. `environments` — build all user environments (parallel) -6. `generate-config` — generate the upstream spack config files for the installed image -7. `modules-done` — generate TCL module files (if `modules.yaml` present) -8. `env-meta` — run `envvars.py uenv` to produce final `meta/env.json` -9. `post-install` — run `post-install-hook` if provided -10. `store.squashfs` — create the final squashfs image - -Key Make.inc rules: -- `%/spack.lock`: concretize a spack environment -- `%/Makefile`: generate a depfile from a lock file (enables parallel package builds) -- `compiler_bin_dirs`: helper to find compiler binaries given install prefixes +2. `pre-install` — run `pre-install-hook` if provided (optional) +3. `mirror-setup` — configure build cache keys +4. `concretize` — `spack -e BUILD_ROOT concretize` (all spec groups in one pass) +5. `install` — `spack -e BUILD_ROOT install` (all groups in dependency order) +6. `compiler-config.yaml` — run `compiler-config.py` to generate packages.yaml with compiler externals +7. `views/NAME` — per-view: generate `activate.sh` via `spack env activate`, then run `envvars.py view` +8. `views` — aggregate target for all views +9. `generate-config` — `make -C generate-config` (writes store/config/{upstreams,packages,repos}.yaml) +10. `modules-done` — `spack module tcl refresh` (if `modules.yaml` present) +11. `env-meta` — run `envvars.py uenv` to produce final `meta/env.json` +12. `post-install` — run `post-install-hook` if provided (optional) +13. `cache-push` — push to build cache (if cache with key configured) +14. `store.squashfs` — create the final squashfs image using the `squashfs` package installed in the `uenv_tools` spec group + +The `uenv_tools` spec group (hardcoded in the `spack.yaml` template) installs `squashfs` as an implicit dependency of gcc, providing the `mksquashfs` binary for image creation. + +All spack commands use `$(SANDBOX) $(SPACK) -e $(BUILD_ROOT)` — there is a single spack environment at the build root. The build runs inside a bwrap sandbox (`bwrap-mutable-root.sh`) that: - Bind-mounts `BUILD/store` → `STORE` (the recipe mount point) - Bind-mounts `BUILD/tmp` → `/tmp` - Puts a tmpfs over `$HOME` (isolates user config) -## Mirrors and Build Caches +## Spec Groups in spack.yaml + +The unified `spack.yaml` uses Spack 1.2 spec groups to express the build order and per-group concretizer settings. Structure: + +- **gcc group**: `explicit: false`, override sets static-library variants for gcc's dependencies (mpc, gmp, mpfr, zstd, zlib) +- **nvhpc/llvm/llvm-amdgpu/intel-oneapi-compilers groups**: `explicit: false`, `needs: [gcc]`, `reuse: false` +- **uenv_tools group**: `explicit: false`, `needs: [gcc]`, installs `squashfs` +- **user environment groups**: `needs: [compiler list]`, override sets `concretizer.unify`, `concretizer.duplicates.strategy`, `packages.all.prefer`, `packages.all.variants`, and `packages.mpi.require` per-environment -Spack mirrors and caches are configured in a single `mirrors.yaml` supplied with `stack-config --mirror ` (see `docs/build-caches.md` for the full reference). It can describe five optional entities: +Per-group `override:` blocks are pushed as the highest-priority config scope during that group's concretization, so `unify` and `duplicates.strategy` settings are truly per-group. -- **`buildcache`** (one): binary cache of built packages — the big build-time speed up. With a `private_key` it signs and pushes packages it builds; without one it is read-only (fetch only). `mount_specific: true` stores binaries in a per-mount-point subdir (Spack binaries embed the install prefix, so each mount point needs its own cache). Packages are pushed per-environment after a successful build; `cuda`, `nvhpc`, `perl` are excluded from pushes. `make cache-force` force-pushes everything built so far. -- **`bootstrap`** (one): for bootstrapping Spack itself (clingo etc.). The `url` is either a local `spack bootstrap mirror` directory (referenced via its own `metadata/sources`+`metadata/binaries`) or a remote url (source-only). Needs **no key** (bootstrap binaries are sha256-verified) → emitted as `config/bootstrap.yaml` (+ a generated `metadata.yaml` only for the remote case); it is NOT a `mirrors.yaml` entry. -- **`sourcemirror`** (many): read-only mirrors providing package source archives. -- **`sourcecache`** (one): a writable local dir Spack fills as it fetches sources → emitted as `config:source_cache`. -- **`concretizer`** (one): a writable local dir persisting Spack's **concretization results** → emitted as `concretizer:concretization_cache:{enable,url}` in `config/concretizer.yaml`. Useful to persist across ephemeral builds. The config key requires Spack ≥ 1.1; Stackinator infers the Spack version from `config.yaml:spack.commit` (via `Recipe.find_spack_version`, defaulting to the latest supported version when the commit can't be pinned) and **skips the cache with a warning** for Spack 1.0 (which rejects the key). +## Build Cache -`sourcecache` is emitted to `config/config.yaml`; `concretizer` to `config/concretizer.yaml`; `bootstrap` to `config/bootstrap.yaml`; `buildcache`/`sourcemirror` to `config/mirrors.yaml` (+ decoded keys under `config/key_store/`). +Optional binary cache configured via YAML file passed to `-c/--cache`: +```yaml +root: /path/to/cache # directory; env vars expanded +key: /path/to/pgp.key # optional; omit for read-only cache +``` -**Legacy:** a binary cache can still be configured with a `cache.yaml` (`root` + optional `key`) passed to `-c/--cache`. This path is deprecated in favour of a `buildcache` entry and will be removed. +Cache is stored in a subdirectory named after the mount point (e.g. `cache/user-environment/`) to avoid relocation issues. Packages are pushed in a single `cache-push` step. Large binary packages (`cuda`, `nvhpc`, `perl`) are excluded from cache pushes. ## Testing @@ -338,13 +328,13 @@ The test coverage is limited — the schema validators and their default-injecti ## Key Invariants and Pitfalls - **Build path restrictions**: cannot be in `/tmp`, `$HOME`, or root `/`. The bwrap sandbox rebinds these. -- **Version 2 is required**: `config.yaml` must have `version: 2` for current `main`. Version 1 recipes require the `releases/v5` branch. -- **gcc is required**: `packages.yaml` in cluster config must define an external `gcc`. It is handled separately from other system packages for the bootstrap build step. +- **Version 3 is required**: `config.yaml` must have `version: 3`. Versions 1 and 2 raise a clear error pointing to the `releases/v6` branch. +- **gcc is required**: cluster `packages.yaml` must define an external `gcc`. It is merged into the global packages.yaml and used by the gcc spec group's override. - **MPI validation**: the MPI name in `network.mpi` must match a key in `network.yaml:mpi` templates from the cluster config. Unknown MPI implementations raise an error. - **View names are globally unique**: view names must be unique across all environments in a recipe. -- **Mirror/cache config comes only from `--mirror`**: never from the recipe or the system/cluster config. A `mirrors.yaml` in the system config dir is rejected with an error. -- **Read-only build cache**: a `buildcache` without a `private_key` is fetched from but never pushed to (`push_to_build_cache` is `None`). -- **Schema-injected defaults**: every non-required field has a `default` in its JSON schema, injected at every level (including `additionalProperties` maps). Rely on fields always being present and check `is None` — do not use `.get()` to guard existence. +- **`mirrors.yaml` in recipes is unsupported**: use `--cache` CLI flag instead. - **`default-view` must exist**: if set in `config.yaml`, the named view must be defined in `environments.yaml` (or be `modules`/`spack`). - **`prefer` is auto-set**: if `null` in the recipe, Stackinator generates a `prefer` constraint using Spack's `%[when=...]` syntax to pin the default compiler. -- **Spack `uenv_tools` environment**: an internal environment named `uenv_tools` is injected into every build to install `squashfs`. Recipe authors must not use this name. +- **`uenv_tools` is reserved**: a spec group named `uenv_tools` is hardcoded in the spack.yaml template to install `squashfs`. Recipe authors must not use this environment name. +- **`packages` field in environments.yaml is ignored**: in v3 recipes, the `packages` list (formerly used to drive `spack external find`) is silently ignored with a warning. Add external packages to `packages.yaml` instead. +- **compiler-config.py must run inside spack python**: it imports `spack.store` and must be invoked as `spack -e BUILD_ROOT python compiler-config.py` to access the correct spack DB. diff --git a/CLAUDE.md.v6 b/CLAUDE.md.v6 new file mode 100644 index 00000000..08d4c68b --- /dev/null +++ b/CLAUDE.md.v6 @@ -0,0 +1,329 @@ +# Stackinator + +Stackinator is a Python CLI tool that generates build configurations for scientific software stacks on HPE Cray EX (Alps) systems. It acts like `cmake`/`configure`: given a **recipe** and a **cluster configuration**, it produces a build directory with Makefiles and Spack YAML files. The actual build is then performed by `make`. + +## Two-Phase Workflow + +``` +stack-config -b BUILD -r RECIPE -s SYSTEM [-c CACHE] [-m MOUNT] + → generates BUILD/ directory with Makefiles + spack.yaml files + +cd BUILD +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make store.squashfs -j64 + → clones Spack, concretises, builds, creates store.squashfs +``` + +The `store.squashfs` SquashFS image is the final artifact, intended to be mounted at the recipe's `store` path (default `/user-environment`) as a uenv image. + +## Repository Layout + +``` +stackinator/ # Python package + main.py # CLI entry point (stack-config) + recipe.py # Recipe class: parses and validates all recipe YAML + builder.py # Builder class: writes all files to the build path + cache.py # Build cache configuration helpers + spack_util.py # Tiny helper: checks if a path is a spack package repo + schema.py # JSON schema validators with default-injection + schema/ # JSON schemas for each YAML file type + config.json + compilers.json + environments.json + cache.json + modules.json + templates/ # Jinja2 templates for all generated files + Makefile # Top-level build orchestration + Makefile.compilers # Compiler build steps + Makefile.environments # Environment build + view generation steps + Makefile.generate-config # Generates upstream spack config for the uenv + Make.user # Build path / store / sandbox variable definitions + repos.yaml # Generated spack repos.yaml + stack-debug.sh # Debug helper script + compilers.*.spack.yaml # Per-compiler spack.yaml configs + environments.spack.yaml # Environment spack.yaml config + etc/ + Make.inc # Shared make rules (concretize, depfile, compiler_bin_dirs) + bwrap-mutable-root.sh # Bubblewrap sandbox wrapper + envvars.py # CLI tool: generates env.json for views and uenv metadata +docs/ # MkDocs documentation source +unittests/ # pytest test suite + test_schema.py # Schema validation tests (primary test coverage) + recipes/ # Example recipes used by tests + yaml/ # Example YAML snippets for testing +``` + +## Recipe Format (input) + +A recipe is a directory containing YAML files: + +### `config.yaml` (required) +```yaml +name: prgenv-gnu +store: /user-environment # mount point; default /user-environment +version: 2 # must be 2 for Spack 1.0 (Stackinator 6+) +spack: + repo: https://github.com/spack/spack.git + commit: releases/v1.0 # branch, tag, or SHA; null = default branch + packages: + repo: https://github.com/spack/spack-packages.git + commit: develop +description: "optional text" +default-view: develop # optional: view loaded when no view is specified +``` + +- `version: 1` (the default) targets Spack v0.23 and is only supported by Stackinator v5 (`releases/v5` branch). **Current `main` requires `version: 2`.** +- `store` can be overridden at configure time with `-m/--mount`. + +### `compilers.yaml` (required) +```yaml +gcc: + version: "13" # required; must be quoted string +nvhpc: # optional + version: "25.1" +llvm: # optional + version: "16" +llvm-amdgpu: # optional + version: "6.0" +intel-oneapi-compilers: # optional + version: "2024.1" +``` + +Build order: `gcc` is built first (using system compiler), then `nvhpc`/`llvm`/`llvm-amdgpu`/`intel-oneapi-compilers` are built using the gcc toolchain. Stackinator appends opinionated variants (e.g. `gcc@13 +bootstrap`, `nvhpc@25.1 ~mpi~blas~lapack`, `llvm@16 +clang ~gold`). + +### `environments.yaml` (required) +```yaml +my-env: + compiler: [gcc] # required; list from compilers.yaml keys; first = default + specs: # required; list of spack specs + - cmake + - hdf5+mpi + network: # optional; null = no MPI + mpi: cray-mpich # full spack spec for MPI (cray-mpich or openmpi) + specs: ['libfabric@1.22'] # optional; overrides network.yaml defaults + unify: true # concretizer: true | false | when_possible (default true) + duplicates: + strategy: minimal # minimal | full | none (default minimal) + deprecated: false # allow deprecated spack versions (default false) + variants: # applied to all packages (packages:all:variants) + - +cuda + - cuda_arch=80 + prefer: null # packages:all:prefer; auto-set if null + packages: # external packages to discover via `spack external find` + - perl + - git + views: # optional filesystem views + default: null # view name → view config (null = defaults) + no-python: + exclude: [python] + uenv: + add_compilers: true # default true; adds compiler symlinks to view/bin + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] + env_vars: + set: + - MYVAR: "value" + - MYVAR2: null # unsets the variable + prepend_path: + - PATH: "/some/path" + append_path: + - PKG_CONFIG_PATH: "/usr/lib/pkgconfig" +``` + +**Key constraints:** +- Do not include MPI or compilers in `specs`; they are handled by `network.mpi` and `compiler`. +- Spec matrices are not supported. +- Only one MPI per environment; create separate environments for multiple MPIs. +- The `prefer` field is auto-generated if `null`: it nudges Spack to use the first compiler for all packages. + +#### Environment variable special syntax +- `${@VAR@}` — deferred expansion: expands `VAR` at uenv load time (e.g. `${@HOME@}`) +- `$@key@` — substitution at configure time: `mount`, `view_name`, `view_path` + +#### Supported prefix-path variables (hardcoded in `etc/envvars.py`) +`ACLOCAL_PATH`, `CMAKE_PREFIX_PATH`, `CPATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, `MANPATH`, `MODULEPATH`, `PATH`, `PKG_CONFIG_PATH`, `PYTHONPATH` + +### `modules.yaml` (optional) +Presence of this file enables module generation. Follows Spack's module config format with two differences: +- `modules:default:arch_folder` must be `false` (Stackinator doesn't support `true`) +- `modules:default:roots:tcl` is ignored and overwritten by Stackinator + +### `packages.yaml` (optional) +Standard Spack `packages.yaml` with recipe-specific external package overrides. + +### `repo/` (optional) +Custom Spack package definitions. Must contain a `packages/` subdirectory. +Merged into a single `alps` namespace repo alongside system and site packages. +Precedence: recipe repo > site repos (from cluster config `repos.yaml`) > Spack builtin. + +### `post-install` / `pre-install` (optional) +Shell scripts (any language) run inside the bwrap sandbox: +- `pre-install`: after Spack is set up, before first compiler build +- `post-install`: after all packages are built, before squashfs generation + +Both are Jinja-templated with variables: `env.mount`, `env.config`, `env.build`, `env.spack`. + +### `extra/` (optional) +Arbitrary files copied to `meta/extra/` in the final image (used for CI metadata). + +## Cluster Configuration (input) + +A directory (passed via `-s/--system`) containing: + +``` +cluster-config/ + packages.yaml # Spack external packages; must include gcc + network.yaml # MPI defaults and network library package configs + repos.yaml # optional; list of relative paths to site-wide spack repos +``` + +`network.yaml` structure: +```yaml +mpi: + cray-mpich: + specs: [libfabric@1.22] # default specs injected when cray-mpich is chosen + openmpi: + specs: [libfabric@2.2.0] +packages: # standard spack packages.yaml content + libfabric: ... + cray-mpich: ... +``` + +Package precedence (recipe.py merges these): recipe `packages.yaml` > `network.yaml` packages > `packages.yaml` (minus gcc). The `gcc` entry from `packages.yaml` is isolated and used only for the gcc compiler build step. + +## Build Directory Structure (output) + +``` +BUILD/ + Makefile # top-level orchestration + Make.user # variables: BUILD_ROOT, STORE, SANDBOX, etc. + Make.inc # shared make rules (copied from etc/) + bwrap-mutable-root.sh # sandbox wrapper (copied from etc/) + envvars.py # view/meta generator (copied from etc/) + spack/ # cloned Spack repository + spack-packages/ # cloned spack-packages repository + config/ # global spack configuration scope + packages.yaml + mirrors.yaml # only if --cache provided + repos.yaml + compilers/ + Makefile + gcc/ + spack.yaml + packages.yaml # generated by spack external find + nvhpc/ # if nvhpc in recipe + spack.yaml + environments/ + Makefile + my-env/ + spack.yaml + generate-config/ # generates the upstream spack config for the final image + Makefile + modules/ # only if modules.yaml in recipe + modules.yaml + store/ # installation root (bind-mounted to recipe.store during build) + meta/ + configure.json # build metadata + env.json.in # view metadata template + recipe/ # copy of the recipe + repos/spack_repo/alps/ # consolidated custom package repo + repos/spack_repo/builtin/ # copy of spack builtin repo + env/ # filesystem views (created during build) + view-name/ + activate.sh + env.json + bin/ lib/ ... + store.squashfs # final compressed image + stack-debug.sh # debug helper: opens shell in build environment +``` + +## Python Architecture + +### `Recipe` class (`recipe.py`) +Parses and validates all recipe inputs in `__init__`. Key responsibilities: +- Validates each YAML file against its JSON schema (with default injection) +- Merges packages from cluster config, network.yaml, and recipe +- Generates full compiler specs (e.g. `gcc@13 +bootstrap`) from `compilers.yaml` +- Processes environments: resolves MPI specs from `network.yaml` templates, sets default `prefer` constraints, builds view metadata +- Provides `compiler_files` and `environment_files` properties (Jinja-rendered Makefiles and spack.yaml files) + +### `Builder` class (`builder.py`) +Writes all files to the build path. Key responsibilities: +- Creates directory structure +- Clones Spack and spack-packages repositories +- Merges and writes the consolidated `alps` spack package repo +- Renders all Jinja templates into build path files +- Writes metadata JSON files + +### `schema.py` +JSON schema validation using `jsonschema`. The `validator()` function extends the validator to auto-inject `default` values from schemas into parsed instances, so downstream code can rely on optional fields always being present. + +### `etc/envvars.py` +A standalone CLI tool (copied into the build directory) with two subcommands: +- `envvars.py view [--compilers] [--prefix_paths]`: reads a Spack-generated `activate.sh`, parses env vars, adds compiler symlinks and prefix paths, writes `env.json` for the view +- `envvars.py uenv [--modules] [--spack]`: merges view `env.json` files with recipe `env_vars` config, writes the final `meta/env.json` + +The `EnvVarSet` class in `envvars.py` is also imported by `recipe.py` for processing `env_vars` at configure time. + +## Build Pipeline (Make targets) + +The top-level `Makefile` orchestrates in order: +1. `spack-setup` — sanity check, bootstrap concretizer +2. `pre-install` — run `pre-install-hook` if provided +3. `mirror-setup` — configure build cache keys +4. `compilers` — build gcc, then nvhpc/llvm/etc. (parallel within each stage) +5. `environments` — build all user environments (parallel) +6. `generate-config` — generate the upstream spack config files for the installed image +7. `modules-done` — generate TCL module files (if `modules.yaml` present) +8. `env-meta` — run `envvars.py uenv` to produce final `meta/env.json` +9. `post-install` — run `post-install-hook` if provided +10. `store.squashfs` — create the final squashfs image + +Key Make.inc rules: +- `%/spack.lock`: concretize a spack environment +- `%/Makefile`: generate a depfile from a lock file (enables parallel package builds) +- `compiler_bin_dirs`: helper to find compiler binaries given install prefixes + +The build runs inside a bwrap sandbox (`bwrap-mutable-root.sh`) that: +- Bind-mounts `BUILD/store` → `STORE` (the recipe mount point) +- Bind-mounts `BUILD/tmp` → `/tmp` +- Puts a tmpfs over `$HOME` (isolates user config) + +## Build Cache + +Optional binary cache configured via YAML file passed to `-c/--cache`: +```yaml +root: /path/to/cache # directory; env vars expanded +key: /path/to/pgp.key # optional; omit for read-only cache +``` + +Cache is stored in a subdirectory named after the mount point (e.g. `cache/user-environment/`) to avoid relocation issues. Packages are pushed per-environment after a successful build. Large binary packages (`cuda`, `nvhpc`, `perl`) are excluded from cache pushes. + +## Testing + +```bash +uv run pytest # run tests +./lint # ruff format + ruff check --fix +``` + +Tests live in `unittests/test_schema.py` and cover schema validation and default injection. Test recipes are in `unittests/recipes/`, example YAML in `unittests/yaml/`. + +The test coverage is limited — the schema validators and their default-injection are well tested, but `Recipe`, `Builder`, and `envvars.py` have minimal test coverage. + +## Code Style + +- Python 3.12+ +- Linting: `ruff` (line length 120, E + F rules, E203 ignored) +- Format: `ruff format` +- Run both via `./lint` + +## Key Invariants and Pitfalls + +- **Build path restrictions**: cannot be in `/tmp`, `$HOME`, or root `/`. The bwrap sandbox rebinds these. +- **Version 2 is required**: `config.yaml` must have `version: 2` for current `main`. Version 1 recipes require the `releases/v5` branch. +- **gcc is required**: `packages.yaml` in cluster config must define an external `gcc`. It is handled separately from other system packages for the bootstrap build step. +- **MPI validation**: the MPI name in `network.mpi` must match a key in `network.yaml:mpi` templates from the cluster config. Unknown MPI implementations raise an error. +- **View names are globally unique**: view names must be unique across all environments in a recipe. +- **`mirrors.yaml` in recipes is unsupported**: use `--cache` CLI flag instead. +- **`default-view` must exist**: if set in `config.yaml`, the named view must be defined in `environments.yaml` (or be `modules`/`spack`). +- **`prefer` is auto-set**: if `null` in the recipe, Stackinator generates a `prefer` constraint using Spack's `%[when=...]` syntax to pin the default compiler. +- **Spack `uenv_tools` environment**: an internal environment named `uenv_tools` is injected into every build to install `squashfs`. Recipe authors must not use this name. diff --git a/docs/building.md b/docs/building.md index 99d2a9a1..9a2915ad 100644 --- a/docs/building.md +++ b/docs/building.md @@ -13,11 +13,24 @@ stack-config --build $BUILD_PATH ... # perform the build cd $BUILD_PATH -env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make modules store.squashfs -j32 +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make modules store.squashfs NJOBS=32 ``` The call to `make` is wrapped with with `env --ignore-env` to unset all environment variables, to improve reproducability of builds. +## Controlling build parallelism + +The number of packages and compilation jobs that Spack runs concurrently is set with the `NJOBS` make variable: + +``` +make store.squashfs NJOBS=64 +``` + +Set `NJOBS` to the number of cores available on the build node (it defaults to `16` if not provided). + +!!! note + Do not use `make -jN` to control build parallelism. The `install` step clears `MAKEFLAGS` before invoking Spack — this avoids a crash with GNU make older than 4.4, whose legacy file-descriptor jobserver Spack mishandles — so the outer `make -j` flag does not reach Spack. `NJOBS` is passed to Spack as `spack install --jobs`, and is the only flag that governs build parallelism. + Build times for stacks typically vary between 30 minutes to 3 hours, depending on the specific packages that have to be built. Using [build caches][ref-mirrors] and building in shared memory (see below) are the most effective methods to speed up builds. diff --git a/stackinator/builder.py b/stackinator/builder.py index 89a14dde..78349a94 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -21,19 +21,12 @@ def install(src, dst, *, ignore=None, symlinks=False): - """Call shutil.copytree or shutil.copy2. copy2 is used if `src` is not a directory. - Afterwards run the equivalent of chmod a+rX dst.""" + """Call shutil.copytree or shutil.copy2, then apply chmod a+rX to dst.""" def apply_permissions_recursive(directory): - """Apply permissions recursively to an entire directory.""" - def set_permissions(path): - """Set permissions for a given path based on chmod a+rX equivalent.""" mode = os.stat(path).st_mode - # Always give read permissions for user, group, and others. new_mode = mode | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH - # If it's a directory or execute bit is set for owner or group, - # set execute bit for all. if stat.S_ISDIR(mode) or mode & (stat.S_IXUSR | stat.S_IXGRP): new_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH os.chmod(path, new_mode) @@ -46,15 +39,9 @@ def set_permissions(path): set_permissions(os.path.join(dirpath, filename)) if stat.S_ISDIR(os.stat(src).st_mode): - shutil.copytree( - src, - dst, - ignore=ignore, - symlinks=symlinks, - ) + shutil.copytree(src, dst, ignore=ignore, symlinks=symlinks) else: shutil.copy2(src, dst, follow_symlinks=symlinks) - # set permissions apply_permissions_recursive(dst) @@ -65,41 +52,28 @@ def __init__(self, args): if not path.is_absolute(): path = pathlib.Path.cwd() / path - # check that if the path exists that it is not a file if path.exists(): if not path.is_dir(): raise IOError("build path is not a directory") parts = path.parts - - # the build path can't be root if len(parts) == 1: raise IOError("build path can't be root '/'") - - # the build path can't be in /tmp because the build step rebinds /tmp. if parts[1] == "tmp": raise IOError("build path can't be in '/tmp'") - - # the build path can't be in $HOME because the build step rebinds $HOME - # NOTE that this would be much easier to determine with PosixPath.is_relative_to - # introduced in Python 3.9. home_parts = pathlib.Path.home().parts if (len(home_parts) <= len(parts)) and (home_parts == parts[: len(home_parts)]): raise IOError("build path can't be in '$HOME' or '~'") - # if path.is_relative_to(pathlib.Path.home()): - # raise IOError("build path can't be in '$HOME' or '~'") self.path = path self.root = pathlib.Path(__file__).parent.resolve() @property def configuration_meta(self): - """Meta data about the configuration and build""" return self._configuration_meta @configuration_meta.setter def configuration_meta(self, recipe): - # generate configuration meta data meta = {} meta["time"] = datetime.now().strftime("%Y%m%d %H:%M:%S") host_data = platform.uname() @@ -123,40 +97,10 @@ def configuration_meta(self, recipe): @property def environment_meta(self): - """The meta data file that describes the environments""" return self._environment_meta @environment_meta.setter def environment_meta(self, recipe): - """ - The output that we want to generate looks like the following, - Which should correspond directly to the environment_view_meta provided - by the recipe. - - { - name: "prgenv-gnu", - description: "useful programming tools", - mount: "/user-environment" - modules: { - "root": /user-environment/modules, - }, - views: { - "default": { - "root": /user-environment/env/default, - "activate": /user-environment/env/default/activate.sh, - "description": "simple devolpment env: compilers, MPI, python, cmake." - "env_vars": { - ... - } - }, - "tools": { - "root": /user-environment/env/tools, - "activate": /user-environment/env/tools/activate.sh, - "description": "handy tools" - } - } - } - """ conf = recipe.config meta = {} meta["name"] = conf["name"] @@ -171,34 +115,24 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): - """Setup the recipe build environment.""" - # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" config_path = self.path / "config" self.path.mkdir(exist_ok=True, parents=True) + env_path = self.path / "env" + env_path.mkdir(exist_ok=True) store_path.mkdir(exist_ok=True) tmp_path.mkdir(exist_ok=True) config_path.mkdir(exist_ok=True) - # check out the version of spack - spack_version = recipe.spack_version - self._logger.debug(f"spack version for templates: {spack_version}") - - # set general build and configuration meta data for the project self.configuration_meta = recipe - - # set the environment view meta data self.environment_meta = recipe - # Clone the spack repository and check out commit if one was given + # Clone spack spack = recipe.config["spack"] - spack_repo = spack["repo"] - spack_commit = spack["commit"] spack_path = self.path / "spack" - - spack_git_commit_result = self._git_clone("spack", spack_repo, spack_commit, spack_path) + spack_git_commit = self._git_clone("spack", spack["repo"], spack["commit"], spack_path) package_repos = recipe.spack_package_repos for pkg_repo in package_repos: @@ -206,13 +140,13 @@ def generate(self, recipe): pkg_repo["commit"] = self._git_clone(pkg_repo["name"], pkg_repo["url"], pkg_repo["ref"], pkg_repo["path"]) spack_meta = { - "url": spack_repo, - "ref": spack_commit, - "commit": spack_git_commit_result, + "url": spack["repo"], + "ref": spack["commit"], + "commit": spack_git_commit, "packages": package_repos, } - # load the jinja templating environment + # Jinja environment for templates template_path = self.root / "templates" jinja_env = jinja2.Environment( loader=jinja2.FileSystemLoader(template_path), @@ -220,6 +154,11 @@ def generate(self, recipe): lstrip_blocks=True, ) + # --- Write the unified spack.yaml --- + with (env_path / "spack.yaml").open("w") as f: + f.write(recipe.spack_yaml) + f.write("\n") + # Write the spack mirror config artifacts (mirrors.yaml, bootstrap config, # and the relocated gpg keys) into the config scope. These were fully # resolved and validated by the recipe, so we just write the bytes. This @@ -229,7 +168,8 @@ def generate(self, recipe): dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(content) - # generate top level makefiles + # --- Write Makefile --- + has_views = any(env_cfg["views"] for env_cfg in recipe.environments.values()) makefile_template = jinja_env.get_template("Makefile") # Extract module types that were configured in recipe.py @@ -245,22 +185,25 @@ def generate(self, recipe): module_types=module_types, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, - spack_version=spack_version, spack_meta=spack_meta, + environments=recipe.environments, + compiler_names=recipe.compiler_names, gpg_keys=recipe.mirrors.gpg_key_paths(config_path), - cache=recipe.build_cache_mirror, + buildcache=recipe.build_cache_mirror, buildcache_push=recipe.push_to_build_cache, exclude_from_cache=["nvhpc", "cuda", "perl"], - verbose=False, + has_views=has_views, + cleanup=recipe.config["cleanup"], + system_gcc=recipe.system_gcc, ) ) f.write("\n") + # --- Write Make.user --- make_user_template = jinja_env.get_template("Make.user") with (self.path / "Make.user").open("w") as f: f.write( make_user_template.render( - spack_version=spack_version, build_path=self.path, store=recipe.mount, no_bwrap=recipe.no_bwrap, @@ -269,11 +212,25 @@ def generate(self, recipe): ) f.write("\n") + # --- Write the sandbox wrapper (binds baked in, self-labelling) --- + sandbox_template = jinja_env.get_template("sandbox") + sandbox_dst = self.path / "sandbox" + with sandbox_dst.open("w") as f: + f.write( + sandbox_template.render( + build_path=self.path, + store=recipe.mount, + no_bwrap=recipe.no_bwrap, + ) + ) + os.chmod(sandbox_dst, os.stat(sandbox_dst).st_mode | stat.S_IEXEC) + + # --- Copy static files from etc/ --- etc_path = self.root / "etc" - for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py"]: + for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py", "compiler-config.py"]: shutil.copy2(etc_path / f_etc, self.path / f_etc) - # used to configure both pre and post install hooks, if they are provided. + # --- Install hooks if provided --- hook_env = { "mount": recipe.mount, "config": recipe.mount / "config", @@ -281,88 +238,51 @@ def generate(self, recipe): "spack": self.path / "spack", } - # copy post install hook file, if provided - post_hook = recipe.post_install_hook - if post_hook is not None: - self._logger.debug("installing post-install-hook script") - jinja_recipe_env = jinja2.Environment(loader=jinja2.FileSystemLoader(recipe.path)) - post_hook_template = jinja_recipe_env.get_template("post-install") - post_hook_destination = store_path / "post-install-hook" - - with post_hook_destination.open("w") as f: - f.write(post_hook_template.render(env=hook_env, verbose=False)) - f.write("\n") - - os.chmod( - post_hook_destination, - os.stat(post_hook_destination).st_mode | stat.S_IEXEC, - ) - - # copy pre install hook file, if provided - pre_hook = recipe.pre_install_hook - if pre_hook is not None: - self._logger.debug("installing pre-install-hook script") - jinja_recipe_env = jinja2.Environment(loader=jinja2.FileSystemLoader(recipe.path)) - pre_hook_template = jinja_recipe_env.get_template("pre-install") - pre_hook_destination = store_path / "pre-install-hook" - - with pre_hook_destination.open("w") as f: - f.write(pre_hook_template.render(env=hook_env, verbose=False)) - f.write("\n") - - os.chmod( - pre_hook_destination, - os.stat(pre_hook_destination).st_mode | stat.S_IEXEC, - ) - - # Generate the system configuration: the compilers, environments, etc. - # that are defined for the target cluster. - packages_path = config_path / "packages.yaml" + for hook_name, hook_src in [ + ("post-install", recipe.post_install_hook), + ("pre-install", recipe.pre_install_hook), + ]: + if hook_src is not None: + self._logger.debug(f"installing {hook_name} script") + jinja_recipe_env = jinja2.Environment(loader=jinja2.FileSystemLoader(recipe.path)) + hook_template = jinja_recipe_env.get_template(hook_src.name) + hook_dst = store_path / f"{hook_name}-hook" + with hook_dst.open("w") as f: + f.write(hook_template.render(env=hook_env, verbose=False)) + f.write("\n") + os.chmod(hook_dst, os.stat(hook_dst).st_mode | stat.S_IEXEC) # the packages.yaml configuration that will be used when building all environments # - the system packages.yaml with gcc removed # - plus additional packages provided by the recipe - global_packages_yaml = yaml.dump(recipe.packages["global"]) - global_packages_path = config_path / "packages.yaml" - with global_packages_path.open("w") as fid: - fid.write(global_packages_yaml) + with (config_path / "packages.yaml").open("w") as f: + f.write(yaml.dump(recipe.packages["build"])) + + config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}}} + with (config_path / "config.yaml").open("w") as f: + f.write(yaml.dump(config_yaml)) # Add custom spack package recipes, configured via Spack repos. - # Step 1: copy Spack repos to store_path where they will be used to - # build the stack, and then be part of the upstream provided - # to users of the stack. - # - # Packages in the recipe are prioritised over cluster specific packages, - # etc. The order of preference from highest to lowest is: - # - # 3. recipe/repo - # 2. cluster-config/repos.yaml - # - if the repos.yaml file exists it will contain a list of relative paths - # to search for package - # 1. package repos from config.yaml in the order specified (typically - # only spack-packages builtin repo) - - # Build a list of repos with packages to install from system config and recipe. + # Build a list of repos with packages to install from system config. + # Packages in the recipe are prioritised over cluster specific packages. + # Order of preference from highest to lowest: + # 3. recipe/repo + # 2. cluster-config/repos.yaml entries + # 1. package repos from config.yaml (e.g. spack-packages builtin) repos = [] # look for repos.yaml file in the system configuration - repo_yaml = recipe.system_config_path / "repos.yaml" - if repo_yaml.exists() and repo_yaml.is_file(): - # open repos.yaml file and reat the list of repos - with repo_yaml.open() as fid: + repo_yaml_path = recipe.system_config_path / "repos.yaml" + if repo_yaml_path.exists() and repo_yaml_path.is_file(): + with repo_yaml_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) - P = raw["repos"] - - self._logger.debug(f"the system configuration has a repo file {repo_yaml} refers to {P}") - - # test each path - for rel_path in P: + for rel_path in raw["repos"]: repo_path = (recipe.system_config_path / rel_path).resolve() if spack_util.is_repo(repo_path): repos.append(repo_path) self._logger.debug(f"adding site spack package repo: {repo_path}") else: - self._logger.error(f"{repo_path} from {repo_yaml} is not a spack package repository") + self._logger.error(f"{repo_path} from {repo_yaml_path} is not a spack package repository") raise RuntimeError("invalid system-provided package repository") self._logger.debug(f"full list of system spack package repos: {repos}") @@ -371,15 +291,10 @@ def generate(self, recipe): # Do this so that incremental builds (though not officially supported) won't break if a repo is updated. repos_path = store_path / "repos" / "spack_repo" repo_dst = repos_path / "alps" - self._logger.debug(f"creating the stack spack repo in {repo_dst}") if repo_dst.exists(): - self._logger.debug(f"{repo_dst} exists ... deleting") shutil.rmtree(repo_dst) - - # create the repository step 1: create the repo directory pkg_dst = repo_dst / "packages" pkg_dst.mkdir(mode=0o755, parents=True) - self._logger.debug(f"created the repo packages path {pkg_dst}") # create the repository step 2: create the repo.yaml file that # configures the alps repo @@ -409,7 +324,6 @@ def generate(self, recipe): self._logger.debug(f" installing recipe package {pkg_path} to {recipe_pkg_dst}") install(pkg_path, dst) - # create the repository step 2: create the repos.yaml file in build_path/config repos_yaml_template = jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: repo_path = recipe.mount / "repos" / "spack_repo" / "alps" @@ -459,101 +373,66 @@ def generate(self, recipe): shutil.rmtree(dst_path) install(src_path, dst_path) - # Generate the makefile and spack.yaml files that describe the compilers - compiler_files = recipe.compiler_files - compiler_path = self.path / "compilers" - compiler_path.mkdir(exist_ok=True) - with (compiler_path / "Makefile").open(mode="w") as f: - f.write(compiler_files["makefile"]) - - for name, files in compiler_files["config"].items(): - compiler_config_path = compiler_path / name - compiler_config_path.mkdir(exist_ok=True) - for file, raw in files.items(): - with (compiler_config_path / file).open(mode="w") as f: - f.write(raw) - - # generate the makefile and spack.yaml files that describe the environments - environment_files = recipe.environment_files - environments_path = self.path / "environments" - os.makedirs(environments_path, exist_ok=True) - with (environments_path / "Makefile").open(mode="w") as f: - f.write(environment_files["makefile"]) - - for name, yml in environment_files["config"].items(): - env_config_path = environments_path / name - env_config_path.mkdir(exist_ok=True) - with (env_config_path / "spack.yaml").open(mode="w") as f: - f.write(yml) - - # generate the makefile that generates the configuration for the spack - # installation in the generate-config sub-directory of the build path. - make_config_template = jinja_env.get_template("Makefile.generate-config") + # --- generate-config subdirectory --- generate_config_path = self.path / "generate-config" generate_config_path.mkdir(exist_ok=True) - # write generate-config/Makefile - all_compilers = [x for x in recipe.compilers.keys()] + make_config_template = jinja_env.get_template("Makefile.generate-config") with (generate_config_path / "Makefile").open("w") as f: f.write( make_config_template.render( modules=recipe.with_modules, build_path=self.path.as_posix(), - all_compilers=all_compilers, - release_compilers=all_compilers, - verbose=False, + compiler_names=recipe.compiler_names, + system_gcc=recipe.system_gcc, ) ) + f.write("\n") + with (generate_config_path / "packages.yaml").open("w") as f: + f.write(yaml.dump(recipe.packages["install"])) - # write modules/modules.yaml + # --- modules --- if recipe.with_modules: - generate_modules_path = self.path / "modules" - generate_modules_path.mkdir(exist_ok=True) - with (generate_modules_path / "modules.yaml").open("w") as f: + modules_path = self.path / "modules" + modules_path.mkdir(exist_ok=True) + with (modules_path / "modules.yaml").open("w") as f: yaml.dump(recipe.modules, f) + with (modules_path / "packages.yaml").open("w") as f: + f.write(yaml.dump(recipe.packages["install"])) - # write the meta data + # --- metadata --- meta_path = store_path / "meta" meta_path.mkdir(exist_ok=True) - # write a json file with basic meta data + with (meta_path / "configure.json").open("w") as f: - # default serialisation is str to serialise the pathlib.PosixPath f.write(json.dumps(self.configuration_meta, sort_keys=True, indent=2, default=str)) f.write("\n") - # write a json file with the environment view meta data with (meta_path / "env.json.in").open("w") as f: - # default serialisation is str to serialise the pathlib.PosixPath f.write(json.dumps(self.environment_meta, sort_keys=True, indent=2, default=str)) f.write("\n") - # copy the recipe to a recipe subdirectory of the meta path meta_recipe_path = meta_path / "recipe" - meta_recipe_path.mkdir(exist_ok=True) if meta_recipe_path.exists(): shutil.rmtree(meta_recipe_path) install(recipe.path, meta_recipe_path, ignore=shutil.ignore_patterns(".git")) - # create the meta/extra path and copy recipe meta data if it exists meta_extra_path = meta_path / "extra" - meta_extra_path.mkdir(exist_ok=True) if meta_extra_path.exists(): shutil.rmtree(meta_extra_path) if recipe.user_extra is not None: - self._logger.debug(f"copying extra recipe meta data to {meta_extra_path}") install(recipe.user_extra, meta_extra_path) + else: + meta_extra_path.mkdir() - # create debug helper script - debug_script_path = self.path / "stack-debug.sh" - debug_script_template = jinja_env.get_template("stack-debug.sh") - with debug_script_path.open("w") as f: + # --- debug helper --- + debug_template = jinja_env.get_template("stack-debug.sh") + with (self.path / "stack-debug.sh").open("w") as f: f.write( - debug_script_template.render( + debug_template.render( mount_path=recipe.mount, build_path=str(self.path), use_bwrap=not recipe.no_bwrap, - spack_version=spack_version, - verbose=False, ) ) f.write("\n") @@ -561,8 +440,6 @@ def generate(self, recipe): def _git_clone(self, name, repo, commit, path): if not (path / ".git").is_dir(): self._logger.info(f"{name}: clone repository {repo} to {path}") - - # clone the repository capture = subprocess.run( ["git", "clone", "--filter=tree:0", repo, path], shell=False, @@ -570,7 +447,6 @@ def _git_clone(self, name, repo, commit, path): stderr=subprocess.STDOUT, ) self._logger.debug(capture.stdout.decode("utf-8")) - if capture.returncode != 0: self._logger.error(f"error cloning the repository {repo}") capture.check_returncode() @@ -578,7 +454,6 @@ def _git_clone(self, name, repo, commit, path): self._logger.info(f"{name}: {repo} already cloned to {path}") if commit: - # Fetch the specific branch self._logger.info(f"{name}: fetching {commit}") capture = subprocess.run( ["git", "-C", path, "fetch", "origin", commit], @@ -587,12 +462,9 @@ def _git_clone(self, name, repo, commit, path): stderr=subprocess.STDOUT, ) self._logger.debug(capture.stdout.decode("utf-8")) - if capture.returncode != 0: - self._logger.debug(f"unable to fetch {commit}") capture.check_returncode() - # Check out the specific branch self._logger.info(f"{name}: checking out {commit}") capture = subprocess.run( ["git", "-C", path, "checkout", commit], @@ -601,15 +473,12 @@ def _git_clone(self, name, repo, commit, path): stderr=subprocess.STDOUT, ) self._logger.debug(capture.stdout.decode("utf-8")) - if capture.returncode != 0: - self._logger.debug(f"unable to change to the requested commit {commit}") capture.check_returncode() else: self._logger.info(f"{name}: no commit set") - # get the commit - git_commit_result = ( + git_commit = ( subprocess.run( ["git", "-C", path, "rev-parse", "HEAD"], shell=False, @@ -619,7 +488,5 @@ def _git_clone(self, name, repo, commit, path): .stdout.strip() .decode("utf-8") ) - - self._logger.info(f"{name}: commit hash is {git_commit_result}") - - return git_commit_result + self._logger.info(f"{name}: commit hash is {git_commit}") + return git_commit diff --git a/stackinator/etc/Make.inc b/stackinator/etc/Make.inc index cf05543d..cc7f9c39 100644 --- a/stackinator/etc/Make.inc +++ b/stackinator/etc/Make.inc @@ -1,8 +1,22 @@ # vi: filetype=make +# Suppress make's recipe echo. Sandboxed commands self-label via the sandbox +# wrapper, and build phases are announced by the banner macro below. +.SILENT: + SPACK ?= spack +SPACK_HELPER := $(SPACK) --color=never + +# Per-phase banners. Colored on a terminal (MAKE_TERMOUT), plain when piped. +ifeq ($(origin NO_COLOR),undefined) +ifneq ($(MAKE_TERMOUT),) +BANNER_COLOR := \033[1;93m +BANNER_RESET := \033[0m +endif +endif -SPACK_ENV = $(SPACK) -e $(dir $@) +# Usage: $(call banner,Human readable phase name) +banner = @printf '\n%b==> [stackinator] %s%b\n' '$(BANNER_COLOR)' '$(1)' '$(BANNER_RESET)' ifndef STORE $(error STORE should point to a Spack install root) @@ -14,19 +28,3 @@ endif store: mkdir -p $(STORE) - -# Concretization -%/spack.lock: %/spack.yaml %/config.yaml %/packages.yaml - /usr/bin/time -f '"%C" took %e seconds.' $(SPACK_ENV) concretize -f - -# Generate Makefiles for the environment install -%/Makefile: %/spack.lock - $(SPACK_ENV) env depfile --make-target-prefix $*/generated -o $@ - -# For generating {compilers,config,packages}.yaml files. -%.yaml: export SPACK_USER_CONFIG_PATH=$(abspath $(dir $@)) -%.yaml: - touch $@ - -# Because Spack doesn't know how to find compilers, we help it by getting the bin folder of gcc, clang, nvc given a install prefix -compiler_bin_dirs = $$(find $(1) '(' -name gcc -o -name clang -o -name nvc -o -name icx ')' -path '*/bin/*' '(' -type f -o -type l ')' -exec dirname {} +) diff --git a/stackinator/etc/compiler-config.py b/stackinator/etc/compiler-config.py new file mode 100644 index 00000000..a6cb44df --- /dev/null +++ b/stackinator/etc/compiler-config.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +""" +Generate or update a packages.yaml file with compiler externals derived from +installed Spack packages. Intended to be run as: + + spack -e BUILD_ROOT python compiler-config.py OUTPUT_YAML COMPILER [COMPILER ...] + +If OUTPUT_YAML already exists (e.g. the build packages.yaml), the compiler +entries are merged in rather than replacing existing content. +""" + +import argparse +import os +import sys + +import yaml + + +_COMPILER_BINS = { + "gcc": [("gcc", "c"), ("g++", "cxx"), ("gfortran", "fortran")], + "llvm": [("clang", "c"), ("clang++", "cxx"), ("flang-new", "fortran")], + "llvm-amdgpu": [("clang", "c"), ("clang++", "cxx"), ("flang-new", "fortran")], + "nvhpc": [("nvc", "c"), ("nvc++", "cxx"), ("nvfortran", "fortran")], + "intel-oneapi-compilers": [("icx", "c"), ("icpx", "cxx"), ("ifx", "fortran")], +} + + +def find_compiler_bins(prefix, compiler_name): + """ + Return a dict mapping language keys (c, cxx, fortran) to absolute binary + paths found under prefix, or None if nothing was found. + """ + candidates = _COMPILER_BINS.get(compiler_name, []) + result = {} + for root, _dirs, files in os.walk(prefix): + for exe, lang in candidates: + if lang not in result and exe in files: + full = os.path.join(root, exe) + if os.access(full, os.X_OK): + result[lang] = full + if len(result) == len(candidates): + break + return result or None + + +def build_compiler_packages(compiler_names): + """ + Query the active Spack DB for each compiler name and return a dict + suitable for merging into packages.yaml. + """ + import spack.store + + packages = {} + for name in compiler_names: + specs = list(spack.store.STORE.db.query(name, explicit=False)) + if not specs: + print(f" compiler-config: no installed specs found for '{name}'", file=sys.stderr) + continue + + externals = [] + for spec in specs: + # Skip externals such as the system gcc. It is registered in the + # cluster packages.yaml and gets pulled into the DB as a bootstrap + # dependency, but only compilers actually built into this environment + # should be surfaced here. The system compiler is included separately, + # and only when selected, via --system-packages. + if spec.external: + continue + prefix = str(spec.prefix) + bins = find_compiler_bins(prefix, name) + if not bins: + print(f" compiler-config: no binaries found for {name} at {prefix}", file=sys.stderr) + continue + externals.append( + { + "spec": f"{spec.name}@{spec.version}", + "prefix": prefix, + "extra_attributes": {"compilers": bins}, + } + ) + print(f" compiler-config: found {name}@{spec.version} at {prefix}", file=sys.stderr) + + if externals: + packages[name] = {"externals": externals, "buildable": False} + + return packages + + +def load_system_compiler_externals(system_packages_path): + """ + Read a packages.yaml and return entries that carry extra_attributes.compilers. + Used to surface system (external) compilers such as system gcc into the output. + """ + with open(system_packages_path) as fid: + data = yaml.safe_load(fid) or {} + + packages = {} + for pkg_name, pkg_data in data.get("packages", {}).items(): + if not isinstance(pkg_data, dict): + continue + compiler_externals = [ + e + for e in pkg_data.get("externals", []) + if isinstance(e, dict) and "extra_attributes" in e and "compilers" in e["extra_attributes"] + ] + if compiler_externals: + packages[pkg_name] = {"externals": compiler_externals, "buildable": False} + print(f" compiler-config: found system compiler '{pkg_name}' in {system_packages_path}", file=sys.stderr) + + return packages + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", help="Path to packages.yaml to create or update") + parser.add_argument("compilers", nargs="*", help="Compiler package names to query") + parser.add_argument( + "--system-packages", + help="Path to a packages.yaml to read system compiler externals from", + default=None, + ) + args = parser.parse_args() + + # Load existing content if the file already exists (merge mode). + existing = {} + if os.path.isfile(args.output): + with open(args.output) as fid: + existing = yaml.safe_load(fid) or {} + + compiler_packages = build_compiler_packages(args.compilers) + + # Pull in any system compiler externals (e.g. system gcc) that carry + # extra_attributes.compilers but are not in the spack store. + if args.system_packages and os.path.isfile(args.system_packages): + system_externals = load_system_compiler_externals(args.system_packages) + for pkg_name, pkg_data in system_externals.items(): + if pkg_name not in compiler_packages: + compiler_packages[pkg_name] = pkg_data + + # Merge: compiler entries overwrite any existing entry for the same package name. + merged = existing.copy() + merged.setdefault("packages", {}).update(compiler_packages) + + with open(args.output, "w") as fid: + yaml.dump(merged, fid, default_flow_style=False) + + print(f" compiler-config: wrote {args.output}", file=sys.stderr) + + +main() diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index fb4c17e1..8994628e 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -476,11 +476,6 @@ def read_activation_script(filename: str, env: Optional[EnvVarSet] = None) -> En def view_impl(args): - print( - f"parsing view {args.root}\n compilers {args.compilers}\n prefix_paths '{args.prefix_paths}'\n \ - build_path '{args.build_path}'" - ) - if not os.path.isdir(args.root): print(f"error - environment root path {args.root} does not exist") exit(1) @@ -503,6 +498,11 @@ def view_impl(args): # remove all prefix path variable values that point to a location inside the build path. envvars.remove_root(args.build_path) + # Canonical symlink names for gcc role keys. For other compiler families + # (nvhpc, llvm, etc.) the binary names are already canonical, so we fall + # back to os.path.basename which gives the right name (nvc, clang, etc.). + _GCC_ROLE_CANONICAL = {"c": "gcc", "cxx": "g++", "fortran": "gfortran"} + if args.compilers is not None: if not os.path.isfile(args.compilers): print(f"error - compiler yaml file {args.compilers} does not exist") @@ -511,28 +511,36 @@ def view_impl(args): with open(args.compilers, "r") as file: data = yaml.safe_load(file) - compilers = [] - for p in data["packages"].values(): - for e in p["externals"]: - if "extra_attributes" in e: - c = e["extra_attributes"]["compilers"] - if c is not None: - compilers.append(c) - - for c in compilers: - source_paths = list(set([os.path.abspath(v) for _, v in c.items() if v is not None])) - target_paths = [os.path.join(bin_path, os.path.basename(f)) for f in source_paths] - for src, dst in zip(source_paths, target_paths): - print(f"creating compiler symlink: {src} -> {dst}") - if os.path.exists(dst): - print(f" first removing {dst}") - os.remove(dst) - os.symlink(src, dst) + # Restrict the symlinked compilers to those wired to this view's + # environment (environments.yaml: compiler). When None, link them all. + allowed = None + if args.compiler_names: + allowed = {n for n in args.compiler_names.split(",") if n} - if args.prefix_paths: - # get the root path of the env - print(f"prefix_paths: searching in {root_path}") + for pkg_name, pkg_data in data["packages"].items(): + if allowed is not None and pkg_name not in allowed: + continue + for e in pkg_data["externals"]: + if "extra_attributes" not in e: + continue + c = e["extra_attributes"].get("compilers") + if not c: + continue + print(f"compiler symlinks: creating for {e.get('prefix', pkg_name)}") + for role, path in c.items(): + if path is None: + continue + src = os.path.abspath(path) + if pkg_name == "gcc": + link_name = _GCC_ROLE_CANONICAL.get(role, os.path.basename(src)) + else: + link_name = os.path.basename(src) + dst = os.path.join(bin_path, link_name) + if os.path.exists(dst): + os.remove(dst) + os.symlink(src, dst) + if args.prefix_paths: for p in args.prefix_paths.split(","): name, value = p.split("=") paths = [] @@ -541,11 +549,9 @@ def view_impl(args): if os.path.isdir(test_path): paths.append(test_path) - print(f"{name}:") - for p in paths: - print(f" {p}") - if len(paths) > 0: + print(f"{name}: {' '.join(paths)}") + if name in envvars.lists: ld_paths = envvars.lists[name].paths final_paths = [p for p in paths if p not in ld_paths] @@ -678,6 +684,13 @@ def meta_impl(args): ) # only add compilers if this argument is passed view_parser.add_argument("--compilers", help="path of the packages.yaml file", type=str, default=None) + view_parser.add_argument( + "--compiler-names", + help="comma-separated compiler package names to symlink into the view; " + "restricts --compilers to the compilers wired to this environment", + type=str, + default=None, + ) uenv_parser = subparsers.add_parser( "uenv", @@ -704,8 +717,6 @@ def meta_impl(args): args = parser.parse_args() if args.command == "uenv": - print("!!! running meta") meta_impl(args) elif args.command == "view": - print("!!! running view") view_impl(args) diff --git a/stackinator/main.py b/stackinator/main.py index 1ae6d617..ea020416 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -136,7 +136,7 @@ def main(): root_logger.info("\nConfiguration finished, run the following to build the environment:\n") root_logger.info(f"cd {builder.path}") root_logger.info( - "env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin HOME=$HOME make store.squashfs -j32" + "env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin HOME=$HOME make store.squashfs NJOBS=32" ) root_logger.info(f"see logfile for more information {logfile}") return 0 diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 8b905be7..374da354 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -107,15 +107,6 @@ def __init__(self, args): ) roots[module_type] = (self.mount / "modules").as_posix() - # DEPRECATED field `config:modules` - if "modules" in self.config: - self._logger.warning("boolean field config.yaml:modules has been deprecated") - - if self.with_modules != self.config["modules"]: - self._logger.error(f"config.yaml:modules:{self.config['modules']}") - self._logger.error(f"modules.yaml:{self.with_modules}") - raise RuntimeError("conflicting modules configuration detected") - self._logger.debug("creating packages") # load recipe/packages.yaml -> recipe_packages (if it exists) @@ -130,22 +121,14 @@ def __init__(self, args): system_packages = {} system_packages_path = self.system_config_path / "packages.yaml" if system_packages_path.is_file(): - # load system yaml with system_packages_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) system_packages = raw["packages"] - # extract gcc packages from system packages - # remove gcc from packages afterwards - if "gcc" in system_packages: - gcc_packages = {"gcc": system_packages["gcc"]} - del system_packages["gcc"] - else: + if "gcc" not in system_packages: raise RuntimeError("The system packages.yaml file does not provide gcc") - # load the optional network.yaml from system config: - # - meta data about mpi - # - package information for network libraries (libfabric, openmpi, cray-mpich, ... etc) + # load the optional network.yaml from system config network_path = self.system_config_path / "network.yaml" network_packages = {} mpi_templates = {} @@ -161,12 +144,17 @@ def __init__(self, args): # note that the order that package sets are specified in is significant. # arguments to the right have higher precedence. - self.packages = { - # the package definition used in every environment - "global": {"packages": system_packages | network_packages | recipe_packages}, - # the package definition used to build gcc (requires system gcc to bootstrap) - "gcc": {"packages": system_packages | gcc_packages | recipe_packages}, - } + # + # build_packages: system + network + recipe packages. + # - system gcc is included here because needed to build the software + # global_packages: system + network + recipe packages. + # - system gcc is only included if building with system gcc + build_packages = system_packages | network_packages | recipe_packages + install_packages = system_packages | network_packages | recipe_packages + if not self.use_system_gcc: + del install_packages["gcc"] + + self.packages = {"build": {"packages": build_packages}, "install": {"packages": install_packages}} # required environments.yaml file environments_path = self.path / "environments.yaml" @@ -176,34 +164,21 @@ def __init__(self, args): with environments_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) - # add a special environment that installs tools required later in the build process. - # currently we only need squashfs for creating the squashfs file. - raw["uenv_tools"] = { - "compiler": ["gcc"], - "network": {"mpi": None, "specs": None}, - "unify": True, - "duplicates": {"strategy": "minimal"}, - "deprecated": False, - "specs": ["squashfs"], - "views": {}, - } schema.EnvironmentsValidator.validate(raw) + self._check_environments_v3(raw) self.generate_environment_specs(raw) # check that the default view exists (if one has been set) self._default_view = self.config["default-view"] if self._default_view is not None: available_views = [view["name"] for env in self.environments.values() for view in env["views"]] - # add the modules and spack views to the list of available views if self.with_modules: available_views.append("modules") available_views.append("spack") if self._default_view not in available_views: - self._logger.error( - f"The default-view {self._default_view} is not the name of a view in the environments.yaml " - "definition (one of {[name for name in available_views]}" + raise RuntimeError( + f"The default-view '{self._default_view}' is not the name of a view defined in environments.yaml" ) - raise RuntimeError("Ivalid default-view in the recipe.") # determine the version of spack being used: it is inferred (best effort) # from the spack commit in config.yaml, defaulting to the latest supported @@ -234,6 +209,15 @@ def __init__(self, args): else: self._logger.debug("no pre install hook provided") + def _check_environments_v3(self, raw): + """Warn about fields that are ignored in v3 recipes.""" + for name, config in raw.items(): + if config and config.get("packages"): + self._logger.warning( + f"environment '{name}': the 'packages' field is ignored in v3 recipes. " + "Add external packages to the recipe's packages.yaml instead." + ) + # Returns: # Path: if the recipe contains a spack package repository # None: if there is the recipe contains no repo @@ -384,107 +368,97 @@ def environment_view_meta(self): "mount": str(self.mount), "view_path": str(view["config"]["root"]), } - env = EnvVarSet.from_envvars(view["extra"]["env_vars"], substitutions) + env_vars = EnvVarSet.from_envvars(view["extra"]["env_vars"], substitutions) except Exception as err: raise RuntimeError(f'In view "{view["name"]}": {err}') view_meta[view["name"]] = { "root": view["config"]["root"], - "description": "", # leave the description empty for now - "recipe_variables": env.as_dict(), + "description": "", + "recipe_variables": env_vars.as_dict(), } return view_meta + @property + def compiler_names(self): + """Names of the compiler packages installed in this recipe (excludes system gcc).""" + return [name for name, c in self.compilers.items() if not c.get("system", False)] + # creates the self.environments field that describes the full specifications # for all of the environments sets, grouped in environments, from the raw # environments.yaml input. def generate_environment_specs(self, raw): environments = raw - # enumerate large binary packages that should not be pushed to binary caches for _, config in environments.items(): config["exclude_from_cache"] = ["cuda", "nvhpc", "perl"] - # check the environment descriptions and ammend where features are missing for name, config in environments.items(): if ("specs" not in config) or (config["specs"] is None): environments[name]["specs"] = [] - # Complete configuration of MPI in each environment - # this involves generate specs for the chosen MPI implementation - # and (optionally) additional dependencies like libfabric, which are - # appended to the list of specs in the environment. + # Resolve MPI specs from the network field for name, config in environments.items(): - # the "mpi" entry records the name of the MPI implementation used by the environment. - # set it to none by default, and have it set if the config["network"] description specifies - # an MPI implementation. environments[name]["mpi"] = None if config["network"]: - # we will build a list of additional specs related to MPI, libfabric, etc to add to the list of specs - # in the generated spack.yaml file. - # start with an empty list: specs = [] if config["network"]["mpi"] is not None: spec = config["network"]["mpi"].strip() - # find the name of the MPI package match = re.match(r"^([A-Za-z][A-Za-z0-9_-]*)", spec) if match: mpi_name = match.group(1) - supported_mpis = [k for k in self.mpi_templates.keys()] + supported_mpis = list(self.mpi_templates.keys()) if mpi_name not in supported_mpis: - raise Exception(f"{mpi_name} is not a supported MPI version: try one of {supported_mpis}.") + raise Exception(f"{mpi_name} is not a supported MPI: try one of {supported_mpis}.") else: raise Exception(f"{spec} is not a valid MPI spec") - # add the mpi spec to the list of explicit specs specs.append(spec) - # if the recipe provided explicit specs for dependencies, inject them: if config["network"]["specs"]: specs += config["network"]["specs"] - # otherwise inject dependencies from network.yaml (if they exist) elif self.mpi_templates[mpi_name]["specs"]: specs += self.mpi_templates[mpi_name]["specs"] environments[name]["mpi"] = mpi_name environments[name]["specs"] += specs - # set constraints that ensure the the main compiler is always used to build packages - # that do not explicitly request a compiler. + # Auto-generate a prefer constraint that pins the default compiler. + # For built compilers, include the version to distinguish from system externals. + # For system gcc, omit the version (there is only one gcc available). for name, config in environments.items(): - # if the recipe provided no "prefer" settings, provide a default one that - # nudges Spack towards using the first compiler (we don't think that this actually - # has much effect). - # With this set, the user can the customise the compiler to use as on a package spec, e.g. - # hdf5+mpi+fortran %fortran=nvhpc - # Which will compile the upstream MPI with nvfortran, as well as downstream dependendencies. if config["prefer"] is None: - compiler = config["compiler"][0] + compiler_key = config["compiler"][0] # spack uses a different name for the intel oneapi compilers # than the package that installs them. - if compiler == "intel-oneapi-compilers": - compiler = "oneapi" + compiler_name = "oneapi" if compiler_key == "intel-oneapi-compilers" else compiler_key + compiler_version = self.compilers[compiler_key].get("version") + versioned = f"{compiler_name}@{compiler_version}" if compiler_version else compiler_name config["prefer"] = [ - f"%[when=%c] c={compiler} %[when=%cxx] cxx={compiler} %[when=%fortran] fortran={compiler}" + f"%[when=%c] c={versioned} %[when=%cxx] cxx={versioned} %[when=%fortran] fortran={versioned}" ] - # Create all meta data for all of the views. + # Compute spec group needs: only compilers with actual (non-system) spec groups. + for name, config in environments.items(): + config["needs"] = [c for c in config["compiler"] if not self.compilers.get(c, {}).get("system", False)] + + # Build view metadata env_names = set() for name, config in environments.items(): views = [] for view_name, vc in config["views"].items(): if view_name in env_names: - raise Exception(f"An environment view with the name '{name}' already exists.") + raise Exception(f"A view named '{view_name}' is defined more than once.") env_names.add(view_name) view_config = copy.deepcopy(vc) # set some default values: - # ["link"] = "roots" - # ["uenv"]["add_compilers"] = True - # ["uenv"]["prefix_paths"] = {} - # ["uenv"]["env_vars"] = {"set": [], "unset": [], "prepend_path": [], "append_path": []} + # view_config["link"] = "roots" + # view_config["uenv"]["add_compilers"] = True + # view_config["uenv"]["prefix_paths"] = {} + # view_config["uenv"]["env_vars"] = {"set": [], "unset": [], "prepend_path": [], "append_path": []} if view_config is None: view_config = {} view_config.setdefault("link", "roots") @@ -501,8 +475,14 @@ def generate_environment_specs(self, raw): [f"{pname}={':'.join(paths)}" for pname, paths in view_config["uenv"]["prefix_paths"].items()] ) view_config["uenv"]["prefix_string"] = prefix_string + # note: root is stored as a string (not pathlib.PosixPath) to avoid + # serialisation issues when the config is written to spack.yaml. view_config["root"] = str(self.mount / "env" / view_name) + # The "uenv" field is stackinator-specific metadata (compiler paths, + # env-var rules) — not a spack view config field. Pop it before + # passing view_config to spack; it travels separately as "extra" and + # is consumed by envvars.py during the view-generation step. extra = view_config.pop("uenv") views.append({"name": view_name, "config": view_config, "extra": extra}) @@ -515,54 +495,30 @@ def generate_environment_specs(self, raw): def generate_compiler_specs(self, raw): compilers = {} - cache_exclude = ["cuda", "nvhpc", "perl"] - gcc = {} - # gcc["packages"] = { - # "external": [ "perl", "m4", "autoconf", "automake", "libtool", "gawk", "python", "texinfo", "gawk", ], - # } gcc_version = raw["gcc"]["version"] - gcc["specs"] = [f"gcc@{gcc_version} + bootstrap"] - gcc["exclude_from_cache"] = [] - - compilers["gcc"] = gcc - - if raw["nvhpc"] is not None: - nvhpc = {} - nvhpc_version = raw["nvhpc"]["version"] - nvhpc["packages"] = False - nvhpc["specs"] = [f"nvhpc@{nvhpc_version} ~mpi~blas~lapack"] - - nvhpc["exclude_from_cache"] = cache_exclude - compilers["nvhpc"] = nvhpc - - if raw["llvm"] is not None: - llvm = {} - llvm_version = raw["llvm"]["version"] - llvm["packages"] = False - llvm["specs"] = [f"llvm@{llvm_version} +clang ~gold"] - - llvm["exclude_from_cache"] = cache_exclude - compilers["llvm"] = llvm - - if raw["llvm-amdgpu"] is not None: - llvm_amdgpu = {} - llvm_amdgpu_version = raw["llvm-amdgpu"]["version"] - llvm_amdgpu["packages"] = False - llvm_amdgpu["specs"] = [f"llvm-amdgpu@{llvm_amdgpu_version}"] - - llvm_amdgpu["exclude_from_cache"] = cache_exclude - compilers["llvm-amdgpu"] = llvm_amdgpu + if gcc_version == "system": + compilers["gcc"] = {"system": True} + else: + compilers["gcc"] = {"specs": [f"gcc@{gcc_version} +bootstrap"], "version": gcc_version} + + for name, spec_template in [ + ("nvhpc", "nvhpc@{version} ~mpi~blas~lapack"), + ("llvm", "llvm@{version} +clang ~gold"), + ("llvm-amdgpu", "llvm-amdgpu@{version}"), + ("intel-oneapi-compilers", "intel-oneapi-compilers@{version}"), + ]: + if raw.get(name) is not None: + version = raw[name]["version"] + compilers[name] = {"specs": [spec_template.format(version=version)], "version": version} - if raw["intel-oneapi-compilers"] is not None: - oneapi = {} - oneapi_version = raw["intel-oneapi-compilers"]["version"] - oneapi["packages"] = False - oneapi["specs"] = [f"intel-oneapi-compilers@{oneapi_version}"] + self.compilers = compilers - oneapi["exclude_from_cache"] = cache_exclude - compilers["intel-oneapi-compilers"] = oneapi + # will the uenv use the system gcc instead of bootstrapping gcc + self.use_system_gcc = (gcc_version == "system") - self.compilers = compilers + @property + def system_gcc(self): + return self.compilers.get("gcc", {}).get("system", False) # The path of the default configuration for the target system/cluster @property @@ -585,56 +541,22 @@ def mount(self): return pathlib.Path(self.config["store"]) @property - def compiler_files(self): - files = {} - + def spack_yaml(self): + """Render the unified spack.yaml for this recipe.""" env = jinja2.Environment( loader=jinja2.FileSystemLoader(self.template_path), trim_blocks=True, lstrip_blocks=True, ) + env.filters["py2yaml"] = schema.py2yaml - makefile_template = env.get_template("Makefile.compilers") - files["makefile"] = makefile_template.render( - compilers=self.compilers, - buildcache=self.push_to_build_cache, - spack_version=self.spack_version, - ) - - files["config"] = {} - for compiler, config in self.compilers.items(): - spack_yaml_template = env.get_template(f"compilers.{compiler}.spack.yaml") - files["config"][compiler] = {} - # compilers//spack.yaml - files["config"][compiler]["spack.yaml"] = spack_yaml_template.render(config=config) - # compilers/gcc/packages.yaml - if compiler == "gcc": - files["config"][compiler]["packages.yaml"] = yaml.dump(self.packages["gcc"]) - - return files - - @property - def environment_files(self): - files = {} + has_views = any(env_cfg["views"] for env_cfg in self.environments.values()) - jenv = jinja2.Environment( - loader=jinja2.FileSystemLoader(self.template_path), - trim_blocks=True, - lstrip_blocks=True, - ) - jenv.filters["py2yaml"] = schema.py2yaml - - makefile_template = jenv.get_template("Makefile.environments") - files["makefile"] = makefile_template.render( + template = env.get_template("spack.yaml") + return template.render( + compilers=self.compilers, environments=self.environments, - buildcache=self.push_to_build_cache, - spack_version=self.spack_version, + store=self.mount, + has_views=has_views, + system_gcc=self.system_gcc, ) - - files["config"] = {} - for env, config in self.environments.items(): - spack_yaml_template = jenv.get_template("environments.spack.yaml") - # generate the spack.yaml file - files["config"][env] = spack_yaml_template.render(config=config, name=env, store=self.mount) - - return files diff --git a/stackinator/schema.py b/stackinator/schema.py index d461ff0e..64cd51a1 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -80,19 +80,18 @@ def validate(self, instance: dict): def check_config_version(instance): rversion = instance.get("version", 1) - if rversion != 2: - if rversion == 1: + if rversion != 3: + if rversion in (1, 2): root_logger.error( - dedent(""" - The recipe is an old version 1 recipe for Spack v0.23 and earlier. - This version of Stackinator supports Spack 1.0, and has deprecated support for Spack v0.23. - Use version 5 of stackinator, which can be accessed via the releases/v5 branch: - git switch releases/v5 + dedent(f""" + The recipe uses version {rversion} of the uenv recipe format. + Stackinator v7 only supports version 3 recipes (Spack 1.2+). - If this recipe is to be used with Spack 1.0, then please add the field 'version: 2' to - config.yaml in your recipe. + To build version {rversion} recipes, use Stackinator v6: + git switch releases/v6 - For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration + To port this recipe to version 3, see: + https://eth-cscs.github.io/stackinator/porting/ """) ) raise RuntimeError("incompatible uenv recipe version") @@ -100,7 +99,7 @@ def check_config_version(instance): root_logger.error( dedent(f""" The config.yaml file sets an unknown recipe version={rversion}. - This version of Stackinator supports version 2 recipes. + Stackinator v7 supports version 3 recipes. For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration """) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 6a9799a1..73fd960f 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -8,13 +8,17 @@ "repo_def": { "type": "object", "additionalProperties": false, - "required": ["repo", "commit"], + "required": ["repo"], "properties": { "repo": { "type": "string" }, "commit": { - "type": "string" + "oneOf": [ + {"type": "string"}, + {"type": "null"} + ], + "default": null }, "path": { "type": "string" @@ -85,7 +89,12 @@ "type": "number", "default": 1, "minimum": 1, - "maximum": 2 + "maximum": 3 + }, + "cleanup" : { + "type": "string", + "enum": ["none", "runtime", "build"], + "default": "none" } } } diff --git a/stackinator/templates/Make.user b/stackinator/templates/Make.user index cb8acb03..833cd441 100644 --- a/stackinator/templates/Make.user +++ b/stackinator/templates/Make.user @@ -5,6 +5,9 @@ # This is the root of the software stack directory. BUILD_ROOT := {{ build_path }} +# Spack environment directory (spack.yaml, packages.yaml, config.yaml live here). +ENV_ROOT := $(BUILD_ROOT)/env + # What Spack should we use? SPACK := spack @@ -15,21 +18,12 @@ SPACK_HELPER := $(SPACK) --color=never # The Spack installation root. STORE := {{ store }} -# When already building inside a sandbox, use `SANDBOX :=` (empty string) -# Without a sandbox, make sure to hide sensitive data such as ~/.ssh through bubblewrap. -# Also bind the directories `./tmp -> /tmp` and `./store -> $(STORE)`, so that -# builds and installs happen inside the current directory. For speed, either -# put the project itself in-memory, or use a flag like --bind /dev/shm/store -# $(STORE). Use `bwrap-mutable-root.sh` in case you need to create a new -# directory at the root /. -{% if no_bwrap %} -SANDBOX := -{% else %} -SANDBOX := $(BUILD_ROOT)/bwrap-mutable-root.sh $\ - --tmpfs ~ $\ - --bind $(BUILD_ROOT)/tmp /tmp $\ - --bind $(BUILD_ROOT)/store $(STORE) -{% endif %} +# Wrapper that runs a command inside the bubblewrap sandbox (binding ./tmp -> +# /tmp and ./store -> $(STORE) and hiding ~). It prints a compact +# `$(BUILD_ROOT)/sandbox ` label so the build output is readable. The bind +# mounts live in the wrapper itself; see $(BUILD_ROOT)/sandbox. When no_bwrap is +# set the wrapper just runs the command directly. +SANDBOX := $(BUILD_ROOT)/sandbox # Makes sure that make -Orecurse continues to print in color. export SPACK_COLOR := always @@ -51,6 +45,13 @@ export SPACK_USER_CACHE_PATH := $(BUILD_ROOT)/cache export SPACK_INSTALL_FLAGS := --verbose {% endif %} +# Number of concurrent build jobs for `spack install`. The new installer treats this as a +# global cap across all packages building at once. +# Set it explicitly because the install recipe clears MAKEFLAGS (see the Makefile), +# so the outer `make -j` no longer reaches spack. +# Override on the build node, e.g. `make store.squashfs NJOBS=64`. +NJOBS ?= 32 + # Reproducibility export LC_ALL := en_US.UTF-8 export TZ := UTC diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 8ba9108f..8fe59def 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -1,17 +1,17 @@ {% set pipejoiner = joiner('|') %} -include Make.user -.PHONY: compilers environments generate-config clean +.PHONY: all generate-config clean -all: environments +all: store.squashfs # Keep track of what Spack version was used. spack-version: $(SANDBOX) $(SPACK) --version > $@ -# Do some sanity checks: (a) are we not on cray, (b) are we using the same -# version as before, (c) ensure that the concretizer is bootstrapped to avoid a -# race where multiple processes start doing that. +# Sanity check: confirm spack works and bootstrap the concretizer. +# +# spack-setup: spack-version @printf "spack arch... " ; \ arch="$$($(SANDBOX) $(SPACK) arch)"; \ @@ -20,7 +20,7 @@ spack-setup: spack-version version="$$($(SANDBOX) $(SPACK) --version)"; \ printf "%s\n" "$$version"; \ printf "bootstrapping spack... "; \ - $(SANDBOX) $(SPACK_HELPER) -d bootstrap now > $(BUILD_ROOT)/spack-bootstrap-output 2>&1; \ + $(SANDBOX) $(SPACK_HELPER) bootstrap now > $(BUILD_ROOT)/spack-bootstrap-output 2>&1; \ if [ "$$?" != "0" ]; then \ printf " failed, see %s\n" $(BUILD_ROOT)/spack-bootstrap-output; \ exit 1; \ @@ -29,58 +29,129 @@ spack-setup: spack-version touch spack-setup pre-install: spack-setup + $(call banner,pre-install hook) +{% if pre_install_hook %} $(SANDBOX) $(STORE)/pre-install-hook +{% else %} + echo "no pre install hook" +{% endif %} + touch pre-install -mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} +mirror-setup: spack-setup pre-install - {% if cache %} + $(call banner,build cache / mirror setup) + {% if buildcache %} @echo "Pulling and trusting keys from configured buildcaches." - $(SANDBOX) $(SPACK) buildcache keys --install --trust + $(SANDBOX) $(SPACK) buildcache keys --install --trust --yes-to-all {% endif %} @echo "Adding mirror gpg keys." {% for key_path in gpg_keys %} - $(SANDBOX) $(SPACK) gpg trust {{ key_path }} + $(SANDBOX) $(SPACK) gpg trust --yes-to-all {{ key_path }} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list touch mirror-setup -compilers: mirror-setup - $(SANDBOX) $(MAKE) -C $@ - -generate-config: compilers - $(SANDBOX) $(MAKE) -C $@ - -environments: compilers - $(SANDBOX) $(MAKE) -C $@ - +concretize: mirror-setup env/spack.yaml + $(call banner,concretize) + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) concretize #--force + touch concretize + +# Clear MAKEFLAGS for the install. When run with `make -j`, GNU make advertises a +# jobserver in MAKEFLAGS. GNU make < 4.4 uses the legacy fd form (`--jobserver-auth=R,W`) +# and closes those fds before running a non-recursive recipe, so they are invalid in the +# spack process - leading to bad file descriptor crashes. +# Hiding MAKEFLAGS makes spack create its own FIFO jobserver sized by `config:build_jobs`. +install: concretize + $(call banner,install packages) + MAKEFLAGS= $(SANDBOX) $(SPACK) -e $(ENV_ROOT) install --jobs $(NJOBS) + touch install + +cache-push: install + $(call banner,push to build cache) +{% if buildcache_push %} + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache create --only=package {{ buildcache }} \ + $$($(SANDBOX) $(SPACK_HELPER) -e $(ENV_ROOT) find --format '{name};{/hash};version={version}' \ + | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ + | grep -v -E 'version=git\.'\ + | cut -d ';' -f2) 2>&1 \ + | awk -v f="$(BUILD_ROOT)/gpg.log" '/^gpg:/{print > f; next} /^==> Fetching/{next} {print; fflush()}' +{% endif %} + touch cache-push + +cleanup: cache-push + $(call banner,garbage collection) +{% if cleanup == "build" %} + $(SANDBOX) $(SPACK) gc --yes-to-all --keep-build-dependencies --except-environment $(ENV_ROOT) +{% elif cleanup == "runtime" %} + $(SANDBOX) $(SPACK) gc --yes-to-all --except-environment $(ENV_ROOT) +{% else %} + echo "no garbage collection requested" +{% endif %} + touch cleanup + +# Generate compiler-config.yaml for use by view and generate-config steps. +compiler-config.yaml: cleanup + $(call banner,generate compiler config) + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ +{% if system_gcc %} + --system-packages=$(BUILD_ROOT)/config/packages.yaml \ +{% endif %} + $(BUILD_ROOT)/compiler-config.yaml{% if compiler_names %} {{ compiler_names | join(' ') }}{% endif %} + +# Generate activate.sh and env.json for each environment view. All views are +# built in a single target so their output is not interleaved under make -j. +views: install compiler-config.yaml +{% for name, config in environments.items() %} +{% for view in config.views %} + $(call banner,view: {{ view.name }}) + $(SANDBOX) mkdir -p $(STORE)/env/{{ view.name }} + $(SANDBOX) sh -c '$(SPACK) env activate -d $(ENV_ROOT) --with-view {{ view.name }} --sh > $(STORE)/env/{{ view.name }}/activate.sh' + $(SANDBOX) $(BUILD_ROOT)/envvars.py view \ + {% if view.extra.add_compilers %}--compilers=$(BUILD_ROOT)/compiler-config.yaml --compiler-names={{ config.compiler | join(',') }} {% endif %}\ + --prefix_paths="{{ view.extra.prefix_string }}" \ + $(STORE)/env/{{ view.name }} \ + $(BUILD_ROOT) +{% endfor %} +{% endfor %} + touch views + +generate-config: install compiler-config.yaml + $(call banner,generate upstream spack config) + $(SANDBOX) $(MAKE) -j1 -C generate-config + +modules-done: generate-config + $(call banner,generate modules) {% if modules %} -modules-done: environments generate-config {% for module_type in module_types %} $(SANDBOX) $(SPACK) -C $(BUILD_ROOT)/modules module {{ module_type }} refresh --upstream-modules --delete-tree --yes-to-all {% endfor %} - touch modules-done +{% else %} + echo "no modules in this uenv" {% endif %} + touch modules-done +env-meta: generate-config views modules-done -env-meta: generate-config environments{% if modules %} modules-done{% endif %} - - $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}'{% for pkg_repo in spack_meta.packages %} --spack-package-repo='{{ pkg_repo.name }},{{ pkg_repo.url }},{{ pkg_repo.ref }},{{ pkg_repo.commit }}'{% endfor %} $(STORE) + $(call banner,generate uenv metadata) + $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv \ + {% if modules %}--modules {% endif %}\ + --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}' \ +{% for pkg_repo in spack_meta.packages %} + --spack-package-repo='{{ pkg_repo.name }},{{ pkg_repo.url }},{{ pkg_repo.ref }},{{ pkg_repo.commit }}' \ +{% endfor %} + $(STORE) touch env-meta post-install: env-meta - {% if post_install_hook %} + $(call banner,post-install hook) +{% if post_install_hook %} $(SANDBOX) $(STORE)/post-install-hook - {% endif %} +{% else %} + echo "no post install hook" +{% endif %} touch post-install -# Create a squashfs file from the installed software. -store.squashfs: post-install - # clean up the __pycache__ paths in the repo - $(SANDBOX) find $(STORE)/repos -type d -name __pycache__ -exec rm -r {} + - $(SANDBOX) chmod -R a+rX $(STORE) - $(SANDBOX) env -u SOURCE_DATE_EPOCH "$$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" $(STORE) $@ -force-uid nobody -force-gid nobody -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3 - # Force push all built packages to the build cache cache-force: mirror-setup {% if buildcache_push %} @@ -100,13 +171,21 @@ cache-force: mirror-setup $(warning "pushing to the build cache is not enabled. See the documentation on how to add a key: https://eth-cscs.github.io/stackinator/build-caches/") {% endif %} -# A backup of all the generated files during the build, useful for posterity, -# excluding the binaries themselves, since they're in the squashfs file -build.tar.gz: spack-version Make.user Make.inc Makefile | environments - tar czf $@ $^ $$(find environments compilers config -maxdepth 2 -name Makefile -o -name '*.yaml') +store.squashfs: env-meta post-install cache-push + + $(call banner,create squashfs image) + $(SANDBOX) find $(STORE)/repos -type d -name __pycache__ -exec rm -r {} + + $(SANDBOX) chmod -R a+rX $(STORE) + $(SANDBOX) env -u SOURCE_DATE_EPOCH \ + "$$($(SANDBOX) $(SPACK_HELPER) -e $(ENV_ROOT) find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" \ + $(STORE) $@ -force-uid nobody -force-gid nobody \ + -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3 -# Clean generate files, does *not* remove installed software. clean: - rm -rf -- $(wildcard */*/spack.lock) $(wildcard */*/.spack-env) $(wildcard */*/Makefile) $(wildcard */*/generated) $(wildcard cache) $(wildcard compilers/*/config.yaml) $(wildcard compilers/*/packages.yaml) $(wildcard environments/*/config.yaml) $(wildcard environments/*/packages.yaml) post-install modules-done env-meta store.squashfs + rm -rf -- spack-version spack-setup{% if pre_install_hook %} pre-install{% endif %} mirror-setup \ + concretize install cleanup{% if push_to_cache and cache.key %} cache-push{% endif %} \ + compiler-config.yaml views generate-config/.done \ + {% if modules %}modules-done {% endif %}env-meta{% if post_install_hook %} post-install{% endif %} \ + store.squashfs spack-bootstrap-output include Make.inc diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers deleted file mode 100644 index 0447863f..00000000 --- a/stackinator/templates/Makefile.compilers +++ /dev/null @@ -1,99 +0,0 @@ -{% set pipejoiner = joiner('|') %} --include ../Make.user - -MAKEFLAGS += --output-sync=recurse - -.PHONY: all .locks .packages.yaml - -all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfor %} - - -# Ensure that spack.lock files are never removed as intermediate files... -.locks:{% for compiler in compilers %} {{ compiler }}/spack.lock{% endfor %} - - -# Ensure that package yaml files are never removed as intermediate files... -.packages.yaml:{% for compiler in compilers %} {{ compiler }}/packages.yaml{% endfor %} - - -{% for compiler, config in compilers.items() %} -{{ compiler }}/generated/build_cache: {{ compiler }}/generated/env -{% if buildcache %} - $(SPACK) -e ./{{ compiler }} buildcache create --only=package {{ buildcache }} \ - $$($(SPACK_HELPER) -e ./{{ compiler }} find --format '{name};{/hash}' \ - | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ - | cut -d ';' -f2) -{% endif %} - touch $@ - -{% endfor %} - -# Configure the install location. -{% for compiler in compilers %}{{ compiler }}/config.yaml {% endfor %}: | store - $(SPACK) config --scope=user add config:install_tree:root:$(STORE) - -# Configure external system dependencies for each compiler toolchain -{% for compiler, config in compilers.items() %} -{% if config.packages and config.packages.external %} -{{ compiler }}/packages.yaml: - $(SPACK) external find --scope=user {% for package in config.packages.external %} {{package}}{% endfor %} - -{% endif %} -{% endfor %} - -{% if compilers.llvm %} -llvm/packages.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --explicit --format '{prefix}' {{ compilers.llvm.requires }})) -{% endif %} - -{% if compilers.get('llvm-amdgpu') %} -llvm-amdgpu/packages.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --explicit --format '{prefix}' {{ compilers.get('llvm-amdgpu').requires }})) -{% endif %} - -{% if compilers.get('intel-oneapi-compilers') %} -intel-oneapi-compilers/packages.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --explicit --format '{prefix}' {{ compilers.get('intel-oneapi-compilers').requires }})) -{% endif %} - -{% if compilers.nvhpc %} -nvhpc/packages.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --explicit --format '{prefix}' {{ compilers.nvhpc.requires }})) -{% endif %} - - - -include ../Make.inc - -# GNU Make isn't very smart about dependencies across included Makefiles, so we -# specify the order here by conditionally including them, when the dependent exists. -ifeq (,$(filter clean,$(MAKECMDGOALS))) - -include gcc/Makefile - -{% if compilers.llvm %} -ifneq (,$(wildcard gcc/Makefile)) -include llvm/Makefile -endif -{% endif %} - -{% if compilers.get('llvm-amdgpu') %} -ifneq (,$(wildcard gcc/Makefile)) -include llvm-amdgpu/Makefile -endif -{% endif %} - -{% if compilers.get('intel-oneapi-compilers') %} -ifneq (,$(wildcard gcc/Makefile)) -include intel-oneapi-compilers/Makefile -endif -{% endif %} - -{% if compilers.nvhpc %} -ifneq (,$(wildcard gcc/Makefile)) -include nvhpc/Makefile -endif -{% endif %} - - -endif diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments deleted file mode 100644 index 74699d46..00000000 --- a/stackinator/templates/Makefile.environments +++ /dev/null @@ -1,67 +0,0 @@ -{% set pipejoiner = joiner('|') %} --include ../Make.user - -MAKEFLAGS += --output-sync=recurse - -.PHONY: all .locks .packages.yaml - -all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} - - -# Ensure that spack.lock files are never removed as intermediate files -.locks:{% for env in environments %} {{ env }}/spack.lock{% endfor %} - -# Ensure that package yaml files are never removed as intermediate files... -.packages.yaml:{% for env in environments %} {{ env }}/packages.yaml{% endfor %} - -# Push built packages to a binary cache if a key has been provided -{% for env, config in environments.items() %} -{{ env }}/generated/build_cache: {{ env }}/generated/view_config -{% if buildcache %} - $(SPACK) --color=never -e ./{{ env }} buildcache create --only=package {{ buildcache }} \ - $$($(SPACK_HELPER) -e ./{{ env }} find --format '{name};{/hash};version={version}' \ - | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ - | grep -v -E 'version=git\.'\ - | cut -d ';' -f2)\ - | grep -Ev '^==> Fetching|^gpg:' -{% endif %} - touch $@ - -{% endfor %} - -# Create environment view where requested -{% for env, config in environments.items() %} -{{ env }}/generated/view_config: {{ env }}/generated/env -{% for view in config.views %} - $(SPACK) env activate --with-view {{ view.name }} --sh ./{{ env }} > $(STORE)/env/{{ view.name }}/activate.sh - $(BUILD_ROOT)/envvars.py view {% if view.extra.add_compilers %}--compilers=./{{ env }}/packages.yaml {% endif %} --prefix_paths="{{ view.extra.prefix_string }}" $(STORE)/env/{{ view.name }} $(BUILD_ROOT) -{% endfor %} - touch $@ - -{% endfor %} - - -{% for env in environments %}{{ env }}/config.yaml {% endfor %}: | store - $(SPACK) config --scope=user add config:install_tree:root:$(STORE) - -# Create the compilers.yaml configuration for each environment -{% for env, config in environments.items() %} -{{ env }}_PREFIX = {% for C in config.compiler %} $$($(SPACK_HELPER) -e ../compilers/{{ C }} find --explicit --format '{prefix}' {{ C.spec }}){% endfor %} - -{{ env }}/packages.yaml: - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $({{ env }}_PREFIX)) -{% if config.packages %} - $(SPACK) external find --not-buildable --scope=user {% for package in config.packages %} {{package}}{% endfor %} -{% endif %} - - -{% endfor %} - --include ../Make.inc - -ifeq (,$(filter clean,$(MAKECMDGOALS))) -{% for env in environments %} -include {{ env }}/Makefile -{% endfor %} -endif - diff --git a/stackinator/templates/Makefile.generate-config b/stackinator/templates/Makefile.generate-config index 1750f037..b23f1394 100644 --- a/stackinator/templates/Makefile.generate-config +++ b/stackinator/templates/Makefile.generate-config @@ -3,38 +3,45 @@ include ../Make.user CONFIG_DIR = $(STORE)/config MODULE_DIR = $(BUILD_ROOT)/modules -# These will be the prefixes of the GCCs, LLVMs and NVHPCs in the respective environments. -ALL_COMPILER_PREFIXES ={% for compiler in all_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --explicit --format='{prefix}' gcc llvm llvm-amdgpu nvhpc intel-oneapi){% endfor %} +all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml{% if modules %} $(MODULE_DIR)/upstreams.yaml{% endif %} +# TODO: the upstreams.yaml files are not getting created in either MODULE_DIR or CONFIG_DIR... whats up with that. -COMPILER_PREFIXES ={% for compiler in release_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --explicit --format='{prefix}' gcc llvm llvm-amdgpu nvhpc intel-oneapi){% endfor %} - - -all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml{% if modules %} $(MODULE_DIR)/upstreams.yaml $(MODULE_DIR)/compilers.yaml{% endif %} +$(CONFIG_DIR)/upstreams.yaml: + $(call banner,generate $(CONFIG_DIR)/upstreams.yaml) + mkdir -p $(CONFIG_DIR) + $(SPACK) -e $(ENV_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) -# Generate the upstream configuration that will be provided by the mounted image -# (requires packages.yaml to ensure that the path $(CONFIG_DIR) has been created). -$(CONFIG_DIR)/upstreams.yaml: $(CONFIG_DIR)/packages.yaml - $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) +{% if modules %} +$(MODULE_DIR)/upstreams.yaml: + $(call banner,generate $(MODULE_DIR)/upstreams.yaml) + $(SPACK) -e $(ENV_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) + +{% endif %} + +# Generate packages.yaml with the spack-built compiler externals so that +# downstream Spack users can find and use the compilers without a compiler.yaml. +# When the recipe uses the system gcc, also pull it in from the build packages.yaml. +# +# TODO: we have replaced this with a simple copy of the packages.yaml used to build +# the software stack +# - without gcc when gcc was built in the uenv +# - with system gcc when the system gcc was used +# maybe this will be sufficient with spack 1.2 for downstream consumers: explore with +# spack-uenv to see whether we still need to explicitly add gcc to the uenv config +# in order for spack to find it. +# +#$(CONFIG_DIR)/packages.yaml: $(CONFIG_DIR)/upstreams.yaml +# $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ +#{% if system_gcc %} +# --system-packages=$(BUILD_ROOT)/config/packages.yaml \ +#{% endif %} +# $(CONFIG_DIR)/packages.yaml{% if compiler_names %} {{ compiler_names | join(' ') }}{% endif %} -# Copy the cluster-specific packages.yaml file to the configuration. $(CONFIG_DIR)/packages.yaml: - # first create the directory, copy the base config and then update with compilers - mkdir -p $(CONFIG_DIR) install -m 644 $(BUILD_ROOT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(COMPILER_PREFIXES)) -# requires packages.yaml to ensure that the path $(CONFIG_DIR) has been created. $(CONFIG_DIR)/repos.yaml: $(CONFIG_DIR)/packages.yaml install -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml -# Generate a configuration used to generate the module files -# The configuration in CONFIG_DIR can't be used for this purpose, because a compilers.yaml -# that includes the bootstrap compiler is required to build the modules. -$(MODULE_DIR)/packages.yaml: # TODO probably this is going to be dropped - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(ALL_COMPILER_PREFIXES)) - -$(MODULE_DIR)/upstreams.yaml: - $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) - include ../Make.inc diff --git a/stackinator/templates/compilers.gcc.spack.yaml b/stackinator/templates/compilers.gcc.spack.yaml deleted file mode 100644 index f8e9ebdc..00000000 --- a/stackinator/templates/compilers.gcc.spack.yaml +++ /dev/null @@ -1,15 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - packages: - gcc: - variants: [build_type=Release +bootstrap +profiled +strip ~binutils] diff --git a/stackinator/templates/compilers.intel-oneapi-compilers.spack.yaml b/stackinator/templates/compilers.intel-oneapi-compilers.spack.yaml deleted file mode 100644 index e567b226..00000000 --- a/stackinator/templates/compilers.intel-oneapi-compilers.spack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - diff --git a/stackinator/templates/compilers.llvm-amdgpu.spack.yaml b/stackinator/templates/compilers.llvm-amdgpu.spack.yaml deleted file mode 100644 index e567b226..00000000 --- a/stackinator/templates/compilers.llvm-amdgpu.spack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - diff --git a/stackinator/templates/compilers.llvm.spack.yaml b/stackinator/templates/compilers.llvm.spack.yaml deleted file mode 100644 index e567b226..00000000 --- a/stackinator/templates/compilers.llvm.spack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - diff --git a/stackinator/templates/compilers.nvhpc.spack.yaml b/stackinator/templates/compilers.nvhpc.spack.yaml deleted file mode 100644 index e567b226..00000000 --- a/stackinator/templates/compilers.nvhpc.spack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - diff --git a/stackinator/templates/environments.spack.yaml b/stackinator/templates/environments.spack.yaml deleted file mode 100644 index 65a11855..00000000 --- a/stackinator/templates/environments.spack.yaml +++ /dev/null @@ -1,42 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - config: - deprecated: {{ config.deprecated }} - concretizer: - unify: {{ config.unify }} - reuse: false - duplicates: - strategy: {{ config.duplicates.strategy }} - specs: -{% for spec in config.specs %} - - '{{ spec }}' -{% endfor %} -{% if config.prefer or config.variants or config.mpi %} - packages: -{% if config.prefer or config.variants %} - all: - {% if config.prefer %} -{% set separator = joiner(', ') %} - prefer: [{% for c in config.prefer %}{{ separator() }}'{{ c }}'{% endfor %}] - {% endif %} - {% if config.variants %} -{% set separator = joiner(', ') %} - variants: [{% for v in config.variants %}{{ separator() }}'{{ v }}'{% endfor %}] - {% endif %} -{% endif %} - {% if config.mpi %} - mpi: - require: '{{ config.mpi }}' - {% endif %} -{% endif %} -{% if config.views %} - view: -{% for view in config.views %} - {{ view.name }}: - {{ view.config|py2yaml(6) }} -{% endfor %} -{% else %} - view: false -{% endif %} diff --git a/stackinator/templates/sandbox b/stackinator/templates/sandbox new file mode 100644 index 00000000..65babbf7 --- /dev/null +++ b/stackinator/templates/sandbox @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Pretty-print the logical command, then run it under the bwrap sandbox. +# The label goes to stderr so that recipes which capture the sandbox output in +# command substitution (e.g. `x="$(SANDBOX spack --version)"`) are unaffected. +if [ -z "${NO_COLOR:-}" ] && [ -t 2 ]; then + printf '\033[38;5;45m%s\033[0m %s\n' "$0" "$*" >&2 +else + printf '%s %s\n' "$0" "$*" >&2 +fi +{% if no_bwrap %} +exec "$@" +{% else %} +exec {{ build_path }}/bwrap-mutable-root.sh \ + --tmpfs ~ \ + --bind {{ build_path }}/tmp /tmp \ + --bind {{ build_path }}/store {{ store }} \ + "$@" +{% endif %} diff --git a/stackinator/templates/spack.yaml b/stackinator/templates/spack.yaml new file mode 100644 index 00000000..9955000b --- /dev/null +++ b/stackinator/templates/spack.yaml @@ -0,0 +1,170 @@ +spack: + concretizer: + reuse: false + specs: + + # ---- Compiler groups ---- +{% if not system_gcc %} + # gcc is built first using the system compiler. + - group: gcc + explicit: false + specs: +{% for spec in compilers.gcc.specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + packages: + gcc: + variants: [build_type=Release +bootstrap +strip ~binutils] + +{% endif %} +{% if compilers.nvhpc %} + - group: nvhpc + explicit: false +{% if not system_gcc %} + needs: [gcc] +{% endif %} + specs: +{% for spec in compilers.nvhpc.specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + +{% endif %} +{% if compilers.llvm %} + - group: llvm + explicit: false +{% if not system_gcc %} + needs: [gcc] +{% endif %} + specs: +{% for spec in compilers.llvm.specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + +{% endif %} +{% if compilers.get('llvm-amdgpu') %} + - group: llvm-amdgpu + explicit: false +{% if not system_gcc %} + needs: [gcc] +{% endif %} + specs: +{% for spec in compilers['llvm-amdgpu'].specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + +{% endif %} +{% if compilers.get('intel-oneapi-compilers') %} + - group: intel-oneapi-compilers + explicit: false +{% if not system_gcc %} + needs: [gcc] +{% endif %} + specs: +{% for spec in compilers['intel-oneapi-compilers'].specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + +{% endif %} + # ---- Internal tools group ---- + # squashfs is required to create the final squashfs image. + - group: uenv_tools + explicit: false +{% if not system_gcc %} + needs: [gcc] +{% endif %} + override: + concretizer: + unify: true + reuse: false + specs: + - squashfs + + # ---- User environment groups ---- +{% for name, config in environments.items() %} + - group: {{ name }} +{% if config.needs %} + needs: [{{ config.needs | join(', ') }}] +{% endif %} + override: + config: + deprecated: {{ config.deprecated | string | lower }} + concretizer: + reuse: false + unify: {{ config.unify | string | lower }} + duplicates: + strategy: {{ config.duplicates.strategy }} +{% if config.prefer or config.variants or config.mpi %} + packages: +{% if config.prefer or config.variants %} + all: +{% if config.prefer %} + prefer: +{% for p in config.prefer %} + - '{{ p }}' +{% endfor %} +{% endif %} +{% if config.variants %} + variants: +{% for v in config.variants %} + - '{{ v }}' +{% endfor %} +{% endif %} +{% endif %} +{% if config.mpi %} + mpi: + require: '{{ config.mpi }}' +{% endif %} +{% endif %} + specs: +{% for spec in config.specs %} + - '{{ spec }}' +{% endfor %} + +{% endfor %} +{% if has_views %} + view: +{% for name, config in environments.items() %} +{% for view in config.views %} + {{ view.name }}: + root: '{{ view.config.root }}' + group: '{{ name }}' + link: '{{ view.config.link }}' +{% if view.config.get('exclude') %} + exclude: +{% for e in view.config.exclude %} + - '{{ e }}' +{% endfor %} +{% endif %} +{% if view.config.get('select') %} + select: +{% for s in view.config.select %} + - '{{ s }}' +{% endfor %} +{% endif %} +{% if view.config.get('projections') %} + projections: + {{ view.config.projections | py2yaml(8) }} +{% endif %} +{% endfor %} +{% endfor %} +{% endif %} diff --git a/unittests/recipes/base-nvgpu/config.yaml b/unittests/recipes/base-nvgpu/config.yaml index 745cb8f9..161e231b 100644 --- a/unittests/recipes/base-nvgpu/config.yaml +++ b/unittests/recipes/base-nvgpu/config.yaml @@ -5,5 +5,4 @@ spack: commit: 6408b51 packages: repo: https://github.com/spack/spack-packages.git - commit: 18b4066 -version: 2 +version: 3 diff --git a/unittests/recipes/cache/config.yaml b/unittests/recipes/cache/config.yaml index d1d9aedd..b878749f 100644 --- a/unittests/recipes/cache/config.yaml +++ b/unittests/recipes/cache/config.yaml @@ -5,5 +5,4 @@ spack: commit: 6408b51 packages: repo: https://github.com/spack/spack-packages.git - commit: 18b4066 -version: 2 +version: 3 diff --git a/unittests/recipes/host-recipe/config.yaml b/unittests/recipes/host-recipe/config.yaml index afa61217..79dfcd26 100644 --- a/unittests/recipes/host-recipe/config.yaml +++ b/unittests/recipes/host-recipe/config.yaml @@ -6,5 +6,4 @@ spack: repo: https://github.com/spack/spack.git packages: repo: https://github.com/spack/spack-packages.git - commit: releases/v0.23 -version: 2 +version: 3 diff --git a/unittests/recipes/with-repo/config.yaml b/unittests/recipes/with-repo/config.yaml index e9a6f54a..1bd5a8a5 100644 --- a/unittests/recipes/with-repo/config.yaml +++ b/unittests/recipes/with-repo/config.yaml @@ -5,5 +5,4 @@ spack: commit: v21.0 packages: repo: https://github.com/spack/spack-packages.git - commit: v21.0 -version: 2 +version: 3 diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 51a320f9..7c6b47a5 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 import pathlib from textwrap import dedent @@ -20,14 +20,19 @@ def yaml_path(test_path): return test_path / "yaml" -@pytest.fixture(params=["host-recipe", "base-nvgpu", "cache", "with-repo", "with-multi-repos"]) -def recipe(request): - return request.param +@pytest.fixture +def recipes(): + return [ + "host-recipe", + "base-nvgpu", + "cache", + "with-repo", + ] @pytest.fixture -def recipe_path(test_path, recipe): - return test_path / "recipes" / recipe +def recipe_paths(test_path, recipes): + return [test_path / "recipes" / r for r in recipes] def test_config_yaml(yaml_path): @@ -37,11 +42,12 @@ def test_config_yaml(yaml_path): schema.ConfigValidator.validate(raw) assert raw["store"] == "/user-environment" assert raw["spack"]["commit"] is None + assert raw["spack"]["packages"]["commit"] is None assert raw["description"] is None - # single repo format with packages commit + # no spack:commit config = dedent(""" - version: 2 + version: 3 name: env-without-spack-commit spack: repo: https://github.com/spack/spack.git @@ -55,22 +61,27 @@ def test_config_yaml(yaml_path): ) schema.ConfigValidator.validate(raw) assert raw["spack"]["commit"] is None - assert raw["spack"]["packages"]["commit"] == "develop-packages" + assert raw["spack"]["packages"]["commit"] is not None assert raw["description"] is None - # single repo format missing packages commit should fail - with pytest.raises(Exception): - config = dedent(""" - version: 2 - name: env-no-pkg-commit - spack: + # no spack:packages:commit + config = dedent(""" + version: 3 + name: env-without-spack-packages-commit + spack: + repo: https://github.com/spack/spack.git + commit: develop + packages: repo: https://github.com/spack/spack.git - commit: develop - packages: - repo: https://github.com/spack/spack.git - """) - raw = yaml.load(config, Loader=yaml.Loader) - schema.ConfigValidator.validate(raw) + """) + raw = yaml.load( + config, + Loader=yaml.Loader, + ) + schema.ConfigValidator.validate(raw) + assert raw["spack"]["commit"] == "develop" + assert raw["spack"]["packages"]["commit"] is None + assert raw["description"] is None # full config with open(yaml_path / "config.full.yaml") as fid: @@ -92,105 +103,13 @@ def test_config_yaml(yaml_path): raw = yaml.load(config, Loader=yaml.Loader) schema.ConfigValidator.validate(raw) - # map format: single entry - config = dedent(""" - version: 2 - name: map-single - spack: - repo: https://github.com/spack/spack.git - packages: - my-packages: - repo: https://github.com/example/spack-packages.git - commit: v1.0 - """) - raw = yaml.load(config, Loader=yaml.Loader) - schema.ConfigValidator.validate(raw) - assert "my-packages" in raw["spack"]["packages"] - assert raw["spack"]["packages"]["my-packages"]["repo"] == "https://github.com/example/spack-packages.git" - assert raw["spack"]["packages"]["my-packages"]["commit"] == "v1.0" - # map format: multiple entries with commits - config = dedent(""" - version: 2 - name: map-multi - spack: - repo: https://github.com/spack/spack.git - packages: - my-packages: - repo: https://github.com/example/spack-packages.git - commit: v1.0 - other-packages: - repo: https://github.com/example/other-packages.git - commit: v2.0 - """) - raw = yaml.load(config, Loader=yaml.Loader) - schema.ConfigValidator.validate(raw) - assert raw["spack"]["packages"]["my-packages"]["commit"] == "v1.0" - assert raw["spack"]["packages"]["other-packages"]["commit"] == "v2.0" - - # map format: empty map should fail - with pytest.raises(Exception): - config = dedent(""" - version: 2 - name: map-empty - spack: - repo: https://github.com/spack/spack.git - packages: {} - """) - raw = yaml.load(config, Loader=yaml.Loader) - schema.ConfigValidator.validate(raw) - - # map format: entry missing repo should fail - with pytest.raises(Exception): - config = dedent(""" - version: 2 - name: map-no-repo - spack: - repo: https://github.com/spack/spack.git - packages: - my-packages: - commit: v1.0 - """) - raw = yaml.load(config, Loader=yaml.Loader) - schema.ConfigValidator.validate(raw) - - # map format: custom path - config = dedent(""" - version: 2 - name: map-custom-path - spack: - repo: https://github.com/spack/spack.git - packages: - my-packages: - repo: https://github.com/example/spack-packages.git - commit: v1.0 - path: custom/repo/location - """) - raw = yaml.load(config, Loader=yaml.Loader) - schema.ConfigValidator.validate(raw) - assert raw["spack"]["packages"]["my-packages"]["path"] == "custom/repo/location" - - # map format: no path (default behavior) - config = dedent(""" - version: 2 - name: map-no-path - spack: - repo: https://github.com/spack/spack.git - packages: - my-packages: - repo: https://github.com/example/spack-packages.git - commit: v2.0 - """) - raw = yaml.load(config, Loader=yaml.Loader) - schema.ConfigValidator.validate(raw) - assert "path" not in raw["spack"]["packages"]["my-packages"] - - -def test_recipe_config_yaml(recipe_path): +def test_recipe_config_yaml(recipe_paths): # validate the config.yaml in the test recipes - with open(recipe_path / "config.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) - schema.ConfigValidator.validate(raw) + for p in recipe_paths: + with open(p / "config.yaml") as fid: + raw = yaml.load(fid, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) def test_compilers_yaml(yaml_path): @@ -209,11 +128,12 @@ def test_compilers_yaml(yaml_path): assert raw["nvhpc"] == {"version": "25.1"} -def test_recipe_compilers_yaml(recipe_path): +def test_recipe_compilers_yaml(recipe_paths): # validate the compilers.yaml in the test recipes - with open(recipe_path / "compilers.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) - schema.CompilersValidator.validate(raw) + for p in recipe_paths: + with open(p / "compilers.yaml") as fid: + raw = yaml.load(fid, Loader=yaml.Loader) + schema.CompilersValidator.validate(raw) def test_environments_yaml(yaml_path): @@ -269,11 +189,12 @@ def test_environments_yaml(yaml_path): schema.EnvironmentsValidator.validate(raw) -def test_recipe_environments_yaml(recipe_path): +def test_recipe_environments_yaml(recipe_paths): # validate the environments.yaml in the test recipes - with open(recipe_path / "environments.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) - schema.EnvironmentsValidator.validate(raw) + for p in recipe_paths: + with open(p / "environments.yaml") as fid: + raw = yaml.load(fid, Loader=yaml.Loader) + schema.EnvironmentsValidator.validate(raw) @pytest.mark.parametrize( diff --git a/unittests/yaml/config.defaults.yaml b/unittests/yaml/config.defaults.yaml index cfd68aa0..3dcdca3b 100644 --- a/unittests/yaml/config.defaults.yaml +++ b/unittests/yaml/config.defaults.yaml @@ -7,6 +7,7 @@ spack: #commit: 6408b51 packages: repo: https://github.com/spack/spack-packages.git - commit: 6408b51 + # default: None == no `git checkout` command + #commit: 6408b51 #modules: True -version: 2 +version: 3 diff --git a/unittests/yaml/config.full.yaml b/unittests/yaml/config.full.yaml index f7891614..31c1f477 100644 --- a/unittests/yaml/config.full.yaml +++ b/unittests/yaml/config.full.yaml @@ -8,4 +8,4 @@ spack: commit: v2025.07.0 modules: False description: "a really useful environment" -version: 2 +version: 3