From a7b46508bff8e3d9c32d834f277dc36c59b8fcf0 Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 18 May 2026 18:12:10 +0200 Subject: [PATCH 01/15] first pass at refactoring for simpler spack 1.2 workflow --- CLAUDE.md | 340 +++++++++++++++ CLAUDE.md.v6 | 329 +++++++++++++++ stackinator/builder.py | 397 +++++------------- stackinator/etc/Make.inc | 19 +- stackinator/etc/compiler-config.py | 106 +++++ stackinator/recipe.py | 245 +++-------- stackinator/schema.py | 21 +- stackinator/schema/config.json | 2 +- stackinator/templates/Makefile | 131 +++--- stackinator/templates/Makefile.compilers | 99 ----- stackinator/templates/Makefile.environments | 67 --- .../templates/Makefile.generate-config | 37 +- .../templates/compilers.gcc.spack.yaml | 25 -- ...ompilers.intel-oneapi-compilers.spack.yaml | 13 - .../compilers.llvm-amdgpu.spack.yaml | 13 - .../templates/compilers.llvm.spack.yaml | 13 - .../templates/compilers.nvhpc.spack.yaml | 13 - stackinator/templates/environments.spack.yaml | 42 -- stackinator/templates/spack.yaml | 171 ++++++++ unittests/recipes/base-nvgpu/config.yaml | 2 +- unittests/recipes/cache/config.yaml | 2 +- unittests/recipes/host-recipe/config.yaml | 2 +- unittests/recipes/with-repo/config.yaml | 2 +- unittests/test_schema.py | 4 +- unittests/yaml/config.defaults.yaml | 2 +- unittests/yaml/config.full.yaml | 2 +- 26 files changed, 1221 insertions(+), 878 deletions(-) create mode 100644 CLAUDE.md create mode 100644 CLAUDE.md.v6 create mode 100644 stackinator/etc/compiler-config.py delete mode 100644 stackinator/templates/Makefile.compilers delete mode 100644 stackinator/templates/Makefile.environments delete mode 100644 stackinator/templates/compilers.gcc.spack.yaml delete mode 100644 stackinator/templates/compilers.intel-oneapi-compilers.spack.yaml delete mode 100644 stackinator/templates/compilers.llvm-amdgpu.spack.yaml delete mode 100644 stackinator/templates/compilers.llvm.spack.yaml delete mode 100644 stackinator/templates/compilers.nvhpc.spack.yaml delete mode 100644 stackinator/templates/environments.spack.yaml create mode 100644 stackinator/templates/spack.yaml diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..e53fd7d7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,340 @@ +# Stackinator + +Stackinator is a Python CLI tool that generates build configurations for scientific software stacks on HPE Cray EX (Alps) systems. It acts like `cmake`/`configure`: given a **recipe** and a **cluster configuration**, it produces a build directory with a Makefile and a single `spack.yaml`. The actual build is then performed by `make`. + +**Stackinator v7 (this branch) supports only version 3 recipes (Spack 1.2+).** For version 2 recipes use the `releases/v6` branch. + +## Two-Phase Workflow + +``` +stack-config -b BUILD -r RECIPE -s SYSTEM [-c CACHE] [-m MOUNT] + → generates BUILD/ directory with Makefile + spack.yaml + +cd BUILD +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make store.squashfs -j64 + → clones Spack, concretises all spec groups, builds, creates store.squashfs +``` + +The `store.squashfs` SquashFS image is the final artifact, intended to be mounted at the recipe's `store` path (default `/user-environment`) as a uenv image. + +## Repository Layout + +``` +stackinator/ # Python package + main.py # CLI entry point (stack-config) + recipe.py # Recipe class: parses and validates all recipe YAML + builder.py # Builder class: writes all files to the build path + cache.py # Build cache configuration helpers + spack_util.py # Tiny helper: checks if a path is a spack package repo + schema.py # JSON schema validators with default-injection + schema/ # JSON schemas for each YAML file type + config.json + compilers.json + environments.json + cache.json + modules.json + templates/ # Jinja2 templates for all generated files + Makefile # Top-level build orchestration (single concretize + install) + Makefile.generate-config # Generates upstream spack config for the uenv + Make.user # Build path / store / sandbox variable definitions + repos.yaml # Generated spack repos.yaml + spack.yaml # Unified spack.yaml with spec groups for all compilers + envs + stack-debug.sh # Debug helper script + etc/ + Make.inc # Shared make rules + bwrap-mutable-root.sh # Bubblewrap sandbox wrapper + compiler-config.py # Spack Python script: generates packages.yaml with compiler externals + envvars.py # CLI tool: generates env.json for views and uenv metadata +docs/ # MkDocs documentation source +unittests/ # pytest test suite + test_schema.py # Schema validation tests (primary test coverage) + recipes/ # Example recipes used by tests + yaml/ # Example YAML snippets for testing +``` + +## Recipe Format (input) + +A recipe is a directory containing YAML files: + +### `config.yaml` (required) +```yaml +name: prgenv-gnu +store: /user-environment # mount point; default /user-environment +version: 3 # must be 3; v1/v2 require Stackinator v6 +spack: + repo: https://github.com/spack/spack.git + commit: releases/v1.2 # branch, tag, or SHA; null = default branch + packages: + repo: https://github.com/spack/spack-packages.git + commit: develop +description: "optional text" +default-view: develop # optional: view loaded when no view is specified +``` + +- `store` can be overridden at configure time with `-m/--mount`. + +### `compilers.yaml` (required) +```yaml +gcc: + version: "13" # required; must be quoted string +nvhpc: # optional + version: "25.1" +llvm: # optional + version: "16" +llvm-amdgpu: # optional + version: "6.0" +intel-oneapi-compilers: # optional + version: "2024.1" +``` + +Build order: `gcc` is built first (using system compiler), then `nvhpc`/`llvm`/`llvm-amdgpu`/`intel-oneapi-compilers` are built using the gcc toolchain. Stackinator appends opinionated variants (e.g. `gcc@13 +bootstrap`, `nvhpc@25.1 ~mpi~blas~lapack`, `llvm@16 +clang ~gold`). Each compiler becomes a separate spec group in the unified `spack.yaml`. + +### `environments.yaml` (required) +```yaml +my-env: + compiler: [gcc] # required; list from compilers.yaml keys; first = default + specs: # required; list of spack specs + - cmake + - hdf5+mpi + network: # optional; null = no MPI + mpi: cray-mpich # MPI implementation name (must match network.yaml key) + specs: ['libfabric@1.22'] # optional; overrides network.yaml defaults + unify: true # concretizer: true | false | when_possible (default true) + duplicates: + strategy: minimal # minimal | full | none (default minimal) + deprecated: false # allow deprecated spack versions (default false) + variants: # applied to all packages (packages:all:variants) + - +cuda + - cuda_arch=80 + prefer: null # packages:all:prefer; auto-set if null + views: # optional filesystem views + default: null # view name → view config (null = defaults) + no-python: + exclude: [python] + uenv: + add_compilers: true # default true; adds compiler symlinks to view/bin + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] + env_vars: + set: + - MYVAR: "value" + - MYVAR2: null # unsets the variable + prepend_path: + - PATH: "/some/path" + append_path: + - PKG_CONFIG_PATH: "/usr/lib/pkgconfig" +``` + +**Key constraints:** +- Do not include MPI or compilers in `specs`; they are handled by `network.mpi` and `compiler`. +- Spec matrices are not supported. +- Only one MPI per environment; create separate environments for multiple MPIs. +- The `prefer` field is auto-generated if `null`: it nudges Spack to use the first compiler for all packages. +- The `packages` field (list of package names for `spack external find`) is accepted by the schema but **ignored in v3 recipes** — a warning is emitted. Add external packages to `packages.yaml` instead. + +#### Environment variable special syntax +- `${@VAR@}` — deferred expansion: expands `VAR` at uenv load time (e.g. `${@HOME@}`) +- `$@key@` — substitution at configure time: `mount`, `view_name`, `view_path` + +#### Supported prefix-path variables (hardcoded in `etc/envvars.py`) +`ACLOCAL_PATH`, `CMAKE_PREFIX_PATH`, `CPATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, `MANPATH`, `MODULEPATH`, `PATH`, `PKG_CONFIG_PATH`, `PYTHONPATH` + +### `modules.yaml` (optional) +Presence of this file enables module generation. Follows Spack's module config format with two differences: +- `modules:default:arch_folder` must be `false` (Stackinator doesn't support `true`) +- `modules:default:roots:tcl` is ignored and overwritten by Stackinator + +### `packages.yaml` (optional) +Standard Spack `packages.yaml` with recipe-specific external package overrides. + +### `repo/` (optional) +Custom Spack package definitions. Must contain a `packages/` subdirectory. +Merged into a single `alps` namespace repo alongside system and site packages. +Precedence: recipe repo > site repos (from cluster config `repos.yaml`) > Spack builtin. + +### `post-install` / `pre-install` (optional) +Shell scripts (any language) run inside the bwrap sandbox: +- `pre-install`: after Spack is set up, before first compiler build +- `post-install`: after all packages are built, before squashfs generation + +Both are Jinja-templated with variables: `env.mount`, `env.config`, `env.build`, `env.spack`. + +### `extra/` (optional) +Arbitrary files copied to `meta/extra/` in the final image (used for CI metadata). + +## Cluster Configuration (input) + +A directory (passed via `-s/--system`) containing: + +``` +cluster-config/ + packages.yaml # Spack external packages; must include gcc + network.yaml # MPI defaults and network library package configs + repos.yaml # optional; list of relative paths to site-wide spack repos +``` + +`network.yaml` structure: +```yaml +mpi: + cray-mpich: + specs: [libfabric@1.22] # default specs injected when cray-mpich is chosen + openmpi: + specs: [libfabric@2.2.0] +packages: # standard spack packages.yaml content + libfabric: ... + cray-mpich: ... +``` + +Package precedence (`recipe.py` merges these): recipe `packages.yaml` > `network.yaml` packages > cluster `packages.yaml`. All packages (including gcc externals) go into a single global `packages.yaml` that is included by the unified `spack.yaml`. + +## Build Directory Structure (output) + +``` +BUILD/ + Makefile # top-level orchestration (single concretize + install) + Make.user # variables: BUILD_ROOT, STORE, SANDBOX, etc. + Make.inc # shared make rules (copied from etc/) + bwrap-mutable-root.sh # sandbox wrapper (copied from etc/) + envvars.py # view/meta generator (copied from etc/) + compiler-config.py # compiler external generator (copied from etc/) + spack.yaml # unified spack.yaml with all spec groups + packages.yaml # merged system + network + recipe packages + config.yaml # spack install tree location + spack/ # cloned Spack repository + spack-packages/ # cloned spack-packages repository + config/ # SPACK_SYSTEM_CONFIG_PATH scope + repos.yaml + mirrors.yaml # only if --cache provided + generate-config/ # generates the upstream spack config for the final image + Makefile + modules/ # only if modules.yaml in recipe + modules.yaml + store/ # installation root (bind-mounted to recipe.store during build) + meta/ + configure.json # build metadata + env.json.in # view metadata template + recipe/ # copy of the recipe + repos/spack_repo/alps/ # consolidated custom package repo + repos/spack_repo/builtin/ # copy of spack builtin repo + env/ # filesystem views (created during build) + view-name/ + activate.sh + env.json + bin/ lib/ ... + store.squashfs # final compressed image + stack-debug.sh # debug helper: opens shell in build environment +``` + +## Python Architecture + +### `Recipe` class (`recipe.py`) +Parses and validates all recipe inputs in `__init__`. Key responsibilities: +- Validates each YAML file against its JSON schema (with default injection) +- Merges packages from cluster config, network.yaml, and recipe into a single dict +- Generates full compiler specs (e.g. `gcc@13 +bootstrap`) from `compilers.yaml` +- Processes environments: resolves MPI specs from `network.yaml` templates, sets default `prefer` constraints, builds view metadata +- Provides `spack_yaml` property (Jinja-rendered unified spack.yaml with spec groups) +- Provides `compiler_names` property (list of compiler package names for `compiler-config.py`) + +### `Builder` class (`builder.py`) +Writes all files to the build path. Key responsibilities: +- Creates directory structure +- Clones Spack and spack-packages repositories +- Merges and writes the consolidated `alps` spack package repo +- Writes the unified `spack.yaml`, `packages.yaml`, and `config.yaml` to `BUILD_ROOT` +- Renders Makefile, Make.user, generate-config/Makefile from Jinja2 templates +- Copies `Make.inc`, `bwrap-mutable-root.sh`, `envvars.py`, `compiler-config.py` from `etc/` +- Writes metadata JSON files + +### `schema.py` +JSON schema validation using `jsonschema`. The `validator()` function extends the validator to auto-inject `default` values from schemas into parsed instances, so downstream code can rely on optional fields always being present. `check_config_version` enforces version 3 and gives a clear error for v1/v2 recipes pointing to the `releases/v6` branch. + +### `etc/compiler-config.py` +Replaces `spack compiler find`. Run as `spack -e BUILD_ROOT python compiler-config.py OUTPUT_YAML COMPILER...`. Uses `spack.store.STORE.db.query()` to find installed compiler packages, walks the install prefix to locate binaries, and writes (or merges into) a `packages.yaml` with `extra_attributes.compilers` entries. This is how compilers become available to downstream Spack users without a `compilers.yaml`. + +### `etc/envvars.py` +A standalone CLI tool (copied into the build directory) with two subcommands: +- `envvars.py view [--compilers=FILE] [--prefix_paths=STR]`: reads a Spack-generated `activate.sh`, parses env vars, adds compiler symlinks and prefix paths, writes `env.json` for the view +- `envvars.py uenv [--modules] [--spack=...]`: merges view `env.json` files with recipe `env_vars` config, writes the final `meta/env.json` + +The `EnvVarSet` class in `envvars.py` is also imported by `recipe.py` for processing `env_vars` at configure time. + +## Build Pipeline (Make targets) + +The top-level `Makefile` orchestrates in order: +1. `spack-setup` — sanity check, bootstrap concretizer +2. `pre-install` — run `pre-install-hook` if provided (optional) +3. `mirror-setup` — configure build cache keys +4. `concretize` — `spack -e BUILD_ROOT concretize` (all spec groups in one pass) +5. `install` — `spack -e BUILD_ROOT install` (all groups in dependency order) +6. `compiler-config.yaml` — run `compiler-config.py` to generate packages.yaml with compiler externals +7. `views/NAME` — per-view: generate `activate.sh` via `spack env activate`, then run `envvars.py view` +8. `views` — aggregate target for all views +9. `generate-config` — `make -C generate-config` (writes store/config/{upstreams,packages,repos}.yaml) +10. `modules-done` — `spack module tcl refresh` (if `modules.yaml` present) +11. `env-meta` — run `envvars.py uenv` to produce final `meta/env.json` +12. `post-install` — run `post-install-hook` if provided (optional) +13. `cache-push` — push to build cache (if cache with key configured) +14. `store.squashfs` — create the final squashfs image using the `squashfs` package installed in the `uenv_tools` spec group + +The `uenv_tools` spec group (hardcoded in the `spack.yaml` template) installs `squashfs` as an implicit dependency of gcc, providing the `mksquashfs` binary for image creation. + +All spack commands use `$(SANDBOX) $(SPACK) -e $(BUILD_ROOT)` — there is a single spack environment at the build root. + +The build runs inside a bwrap sandbox (`bwrap-mutable-root.sh`) that: +- Bind-mounts `BUILD/store` → `STORE` (the recipe mount point) +- Bind-mounts `BUILD/tmp` → `/tmp` +- Puts a tmpfs over `$HOME` (isolates user config) + +## Spec Groups in spack.yaml + +The unified `spack.yaml` uses Spack 1.2 spec groups to express the build order and per-group concretizer settings. Structure: + +- **gcc group**: `explicit: false`, override sets static-library variants for gcc's dependencies (mpc, gmp, mpfr, zstd, zlib) +- **nvhpc/llvm/llvm-amdgpu/intel-oneapi-compilers groups**: `explicit: false`, `needs: [gcc]`, `reuse: false` +- **uenv_tools group**: `explicit: false`, `needs: [gcc]`, installs `squashfs` +- **user environment groups**: `needs: [compiler list]`, override sets `concretizer.unify`, `concretizer.duplicates.strategy`, `packages.all.prefer`, `packages.all.variants`, and `packages.mpi.require` per-environment + +Per-group `override:` blocks are pushed as the highest-priority config scope during that group's concretization, so `unify` and `duplicates.strategy` settings are truly per-group. + +## Build Cache + +Optional binary cache configured via YAML file passed to `-c/--cache`: +```yaml +root: /path/to/cache # directory; env vars expanded +key: /path/to/pgp.key # optional; omit for read-only cache +``` + +Cache is stored in a subdirectory named after the mount point (e.g. `cache/user-environment/`) to avoid relocation issues. Packages are pushed in a single `cache-push` step. Large binary packages (`cuda`, `nvhpc`, `perl`) are excluded from cache pushes. + +## Testing + +```bash +uv run pytest # run tests +./lint # ruff format + ruff check --fix +``` + +Tests live in `unittests/test_schema.py` and cover schema validation and default injection. Test recipes are in `unittests/recipes/`, example YAML in `unittests/yaml/`. + +The test coverage is limited — the schema validators and their default-injection are well tested, but `Recipe`, `Builder`, and `envvars.py` have minimal test coverage. + +## Code Style + +- Python 3.12+ +- Linting: `ruff` (line length 120, E + F rules, E203 ignored) +- Format: `ruff format` +- Run both via `./lint` + +## Key Invariants and Pitfalls + +- **Build path restrictions**: cannot be in `/tmp`, `$HOME`, or root `/`. The bwrap sandbox rebinds these. +- **Version 3 is required**: `config.yaml` must have `version: 3`. Versions 1 and 2 raise a clear error pointing to the `releases/v6` branch. +- **gcc is required**: cluster `packages.yaml` must define an external `gcc`. It is merged into the global packages.yaml and used by the gcc spec group's override. +- **MPI validation**: the MPI name in `network.mpi` must match a key in `network.yaml:mpi` templates from the cluster config. Unknown MPI implementations raise an error. +- **View names are globally unique**: view names must be unique across all environments in a recipe. +- **`mirrors.yaml` in recipes is unsupported**: use `--cache` CLI flag instead. +- **`default-view` must exist**: if set in `config.yaml`, the named view must be defined in `environments.yaml` (or be `modules`/`spack`). +- **`prefer` is auto-set**: if `null` in the recipe, Stackinator generates a `prefer` constraint using Spack's `%[when=...]` syntax to pin the default compiler. +- **`uenv_tools` is reserved**: a spec group named `uenv_tools` is hardcoded in the spack.yaml template to install `squashfs`. Recipe authors must not use this environment name. +- **`packages` field in environments.yaml is ignored**: in v3 recipes, the `packages` list (formerly used to drive `spack external find`) is silently ignored with a warning. Add external packages to `packages.yaml` instead. +- **compiler-config.py must run inside spack python**: it imports `spack.store` and must be invoked as `spack -e BUILD_ROOT python compiler-config.py` to access the correct spack DB. diff --git a/CLAUDE.md.v6 b/CLAUDE.md.v6 new file mode 100644 index 00000000..08d4c68b --- /dev/null +++ b/CLAUDE.md.v6 @@ -0,0 +1,329 @@ +# Stackinator + +Stackinator is a Python CLI tool that generates build configurations for scientific software stacks on HPE Cray EX (Alps) systems. It acts like `cmake`/`configure`: given a **recipe** and a **cluster configuration**, it produces a build directory with Makefiles and Spack YAML files. The actual build is then performed by `make`. + +## Two-Phase Workflow + +``` +stack-config -b BUILD -r RECIPE -s SYSTEM [-c CACHE] [-m MOUNT] + → generates BUILD/ directory with Makefiles + spack.yaml files + +cd BUILD +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make store.squashfs -j64 + → clones Spack, concretises, builds, creates store.squashfs +``` + +The `store.squashfs` SquashFS image is the final artifact, intended to be mounted at the recipe's `store` path (default `/user-environment`) as a uenv image. + +## Repository Layout + +``` +stackinator/ # Python package + main.py # CLI entry point (stack-config) + recipe.py # Recipe class: parses and validates all recipe YAML + builder.py # Builder class: writes all files to the build path + cache.py # Build cache configuration helpers + spack_util.py # Tiny helper: checks if a path is a spack package repo + schema.py # JSON schema validators with default-injection + schema/ # JSON schemas for each YAML file type + config.json + compilers.json + environments.json + cache.json + modules.json + templates/ # Jinja2 templates for all generated files + Makefile # Top-level build orchestration + Makefile.compilers # Compiler build steps + Makefile.environments # Environment build + view generation steps + Makefile.generate-config # Generates upstream spack config for the uenv + Make.user # Build path / store / sandbox variable definitions + repos.yaml # Generated spack repos.yaml + stack-debug.sh # Debug helper script + compilers.*.spack.yaml # Per-compiler spack.yaml configs + environments.spack.yaml # Environment spack.yaml config + etc/ + Make.inc # Shared make rules (concretize, depfile, compiler_bin_dirs) + bwrap-mutable-root.sh # Bubblewrap sandbox wrapper + envvars.py # CLI tool: generates env.json for views and uenv metadata +docs/ # MkDocs documentation source +unittests/ # pytest test suite + test_schema.py # Schema validation tests (primary test coverage) + recipes/ # Example recipes used by tests + yaml/ # Example YAML snippets for testing +``` + +## Recipe Format (input) + +A recipe is a directory containing YAML files: + +### `config.yaml` (required) +```yaml +name: prgenv-gnu +store: /user-environment # mount point; default /user-environment +version: 2 # must be 2 for Spack 1.0 (Stackinator 6+) +spack: + repo: https://github.com/spack/spack.git + commit: releases/v1.0 # branch, tag, or SHA; null = default branch + packages: + repo: https://github.com/spack/spack-packages.git + commit: develop +description: "optional text" +default-view: develop # optional: view loaded when no view is specified +``` + +- `version: 1` (the default) targets Spack v0.23 and is only supported by Stackinator v5 (`releases/v5` branch). **Current `main` requires `version: 2`.** +- `store` can be overridden at configure time with `-m/--mount`. + +### `compilers.yaml` (required) +```yaml +gcc: + version: "13" # required; must be quoted string +nvhpc: # optional + version: "25.1" +llvm: # optional + version: "16" +llvm-amdgpu: # optional + version: "6.0" +intel-oneapi-compilers: # optional + version: "2024.1" +``` + +Build order: `gcc` is built first (using system compiler), then `nvhpc`/`llvm`/`llvm-amdgpu`/`intel-oneapi-compilers` are built using the gcc toolchain. Stackinator appends opinionated variants (e.g. `gcc@13 +bootstrap`, `nvhpc@25.1 ~mpi~blas~lapack`, `llvm@16 +clang ~gold`). + +### `environments.yaml` (required) +```yaml +my-env: + compiler: [gcc] # required; list from compilers.yaml keys; first = default + specs: # required; list of spack specs + - cmake + - hdf5+mpi + network: # optional; null = no MPI + mpi: cray-mpich # full spack spec for MPI (cray-mpich or openmpi) + specs: ['libfabric@1.22'] # optional; overrides network.yaml defaults + unify: true # concretizer: true | false | when_possible (default true) + duplicates: + strategy: minimal # minimal | full | none (default minimal) + deprecated: false # allow deprecated spack versions (default false) + variants: # applied to all packages (packages:all:variants) + - +cuda + - cuda_arch=80 + prefer: null # packages:all:prefer; auto-set if null + packages: # external packages to discover via `spack external find` + - perl + - git + views: # optional filesystem views + default: null # view name → view config (null = defaults) + no-python: + exclude: [python] + uenv: + add_compilers: true # default true; adds compiler symlinks to view/bin + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] + env_vars: + set: + - MYVAR: "value" + - MYVAR2: null # unsets the variable + prepend_path: + - PATH: "/some/path" + append_path: + - PKG_CONFIG_PATH: "/usr/lib/pkgconfig" +``` + +**Key constraints:** +- Do not include MPI or compilers in `specs`; they are handled by `network.mpi` and `compiler`. +- Spec matrices are not supported. +- Only one MPI per environment; create separate environments for multiple MPIs. +- The `prefer` field is auto-generated if `null`: it nudges Spack to use the first compiler for all packages. + +#### Environment variable special syntax +- `${@VAR@}` — deferred expansion: expands `VAR` at uenv load time (e.g. `${@HOME@}`) +- `$@key@` — substitution at configure time: `mount`, `view_name`, `view_path` + +#### Supported prefix-path variables (hardcoded in `etc/envvars.py`) +`ACLOCAL_PATH`, `CMAKE_PREFIX_PATH`, `CPATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, `MANPATH`, `MODULEPATH`, `PATH`, `PKG_CONFIG_PATH`, `PYTHONPATH` + +### `modules.yaml` (optional) +Presence of this file enables module generation. Follows Spack's module config format with two differences: +- `modules:default:arch_folder` must be `false` (Stackinator doesn't support `true`) +- `modules:default:roots:tcl` is ignored and overwritten by Stackinator + +### `packages.yaml` (optional) +Standard Spack `packages.yaml` with recipe-specific external package overrides. + +### `repo/` (optional) +Custom Spack package definitions. Must contain a `packages/` subdirectory. +Merged into a single `alps` namespace repo alongside system and site packages. +Precedence: recipe repo > site repos (from cluster config `repos.yaml`) > Spack builtin. + +### `post-install` / `pre-install` (optional) +Shell scripts (any language) run inside the bwrap sandbox: +- `pre-install`: after Spack is set up, before first compiler build +- `post-install`: after all packages are built, before squashfs generation + +Both are Jinja-templated with variables: `env.mount`, `env.config`, `env.build`, `env.spack`. + +### `extra/` (optional) +Arbitrary files copied to `meta/extra/` in the final image (used for CI metadata). + +## Cluster Configuration (input) + +A directory (passed via `-s/--system`) containing: + +``` +cluster-config/ + packages.yaml # Spack external packages; must include gcc + network.yaml # MPI defaults and network library package configs + repos.yaml # optional; list of relative paths to site-wide spack repos +``` + +`network.yaml` structure: +```yaml +mpi: + cray-mpich: + specs: [libfabric@1.22] # default specs injected when cray-mpich is chosen + openmpi: + specs: [libfabric@2.2.0] +packages: # standard spack packages.yaml content + libfabric: ... + cray-mpich: ... +``` + +Package precedence (recipe.py merges these): recipe `packages.yaml` > `network.yaml` packages > `packages.yaml` (minus gcc). The `gcc` entry from `packages.yaml` is isolated and used only for the gcc compiler build step. + +## Build Directory Structure (output) + +``` +BUILD/ + Makefile # top-level orchestration + Make.user # variables: BUILD_ROOT, STORE, SANDBOX, etc. + Make.inc # shared make rules (copied from etc/) + bwrap-mutable-root.sh # sandbox wrapper (copied from etc/) + envvars.py # view/meta generator (copied from etc/) + spack/ # cloned Spack repository + spack-packages/ # cloned spack-packages repository + config/ # global spack configuration scope + packages.yaml + mirrors.yaml # only if --cache provided + repos.yaml + compilers/ + Makefile + gcc/ + spack.yaml + packages.yaml # generated by spack external find + nvhpc/ # if nvhpc in recipe + spack.yaml + environments/ + Makefile + my-env/ + spack.yaml + generate-config/ # generates the upstream spack config for the final image + Makefile + modules/ # only if modules.yaml in recipe + modules.yaml + store/ # installation root (bind-mounted to recipe.store during build) + meta/ + configure.json # build metadata + env.json.in # view metadata template + recipe/ # copy of the recipe + repos/spack_repo/alps/ # consolidated custom package repo + repos/spack_repo/builtin/ # copy of spack builtin repo + env/ # filesystem views (created during build) + view-name/ + activate.sh + env.json + bin/ lib/ ... + store.squashfs # final compressed image + stack-debug.sh # debug helper: opens shell in build environment +``` + +## Python Architecture + +### `Recipe` class (`recipe.py`) +Parses and validates all recipe inputs in `__init__`. Key responsibilities: +- Validates each YAML file against its JSON schema (with default injection) +- Merges packages from cluster config, network.yaml, and recipe +- Generates full compiler specs (e.g. `gcc@13 +bootstrap`) from `compilers.yaml` +- Processes environments: resolves MPI specs from `network.yaml` templates, sets default `prefer` constraints, builds view metadata +- Provides `compiler_files` and `environment_files` properties (Jinja-rendered Makefiles and spack.yaml files) + +### `Builder` class (`builder.py`) +Writes all files to the build path. Key responsibilities: +- Creates directory structure +- Clones Spack and spack-packages repositories +- Merges and writes the consolidated `alps` spack package repo +- Renders all Jinja templates into build path files +- Writes metadata JSON files + +### `schema.py` +JSON schema validation using `jsonschema`. The `validator()` function extends the validator to auto-inject `default` values from schemas into parsed instances, so downstream code can rely on optional fields always being present. + +### `etc/envvars.py` +A standalone CLI tool (copied into the build directory) with two subcommands: +- `envvars.py view [--compilers] [--prefix_paths]`: reads a Spack-generated `activate.sh`, parses env vars, adds compiler symlinks and prefix paths, writes `env.json` for the view +- `envvars.py uenv [--modules] [--spack]`: merges view `env.json` files with recipe `env_vars` config, writes the final `meta/env.json` + +The `EnvVarSet` class in `envvars.py` is also imported by `recipe.py` for processing `env_vars` at configure time. + +## Build Pipeline (Make targets) + +The top-level `Makefile` orchestrates in order: +1. `spack-setup` — sanity check, bootstrap concretizer +2. `pre-install` — run `pre-install-hook` if provided +3. `mirror-setup` — configure build cache keys +4. `compilers` — build gcc, then nvhpc/llvm/etc. (parallel within each stage) +5. `environments` — build all user environments (parallel) +6. `generate-config` — generate the upstream spack config files for the installed image +7. `modules-done` — generate TCL module files (if `modules.yaml` present) +8. `env-meta` — run `envvars.py uenv` to produce final `meta/env.json` +9. `post-install` — run `post-install-hook` if provided +10. `store.squashfs` — create the final squashfs image + +Key Make.inc rules: +- `%/spack.lock`: concretize a spack environment +- `%/Makefile`: generate a depfile from a lock file (enables parallel package builds) +- `compiler_bin_dirs`: helper to find compiler binaries given install prefixes + +The build runs inside a bwrap sandbox (`bwrap-mutable-root.sh`) that: +- Bind-mounts `BUILD/store` → `STORE` (the recipe mount point) +- Bind-mounts `BUILD/tmp` → `/tmp` +- Puts a tmpfs over `$HOME` (isolates user config) + +## Build Cache + +Optional binary cache configured via YAML file passed to `-c/--cache`: +```yaml +root: /path/to/cache # directory; env vars expanded +key: /path/to/pgp.key # optional; omit for read-only cache +``` + +Cache is stored in a subdirectory named after the mount point (e.g. `cache/user-environment/`) to avoid relocation issues. Packages are pushed per-environment after a successful build. Large binary packages (`cuda`, `nvhpc`, `perl`) are excluded from cache pushes. + +## Testing + +```bash +uv run pytest # run tests +./lint # ruff format + ruff check --fix +``` + +Tests live in `unittests/test_schema.py` and cover schema validation and default injection. Test recipes are in `unittests/recipes/`, example YAML in `unittests/yaml/`. + +The test coverage is limited — the schema validators and their default-injection are well tested, but `Recipe`, `Builder`, and `envvars.py` have minimal test coverage. + +## Code Style + +- Python 3.12+ +- Linting: `ruff` (line length 120, E + F rules, E203 ignored) +- Format: `ruff format` +- Run both via `./lint` + +## Key Invariants and Pitfalls + +- **Build path restrictions**: cannot be in `/tmp`, `$HOME`, or root `/`. The bwrap sandbox rebinds these. +- **Version 2 is required**: `config.yaml` must have `version: 2` for current `main`. Version 1 recipes require the `releases/v5` branch. +- **gcc is required**: `packages.yaml` in cluster config must define an external `gcc`. It is handled separately from other system packages for the bootstrap build step. +- **MPI validation**: the MPI name in `network.mpi` must match a key in `network.yaml:mpi` templates from the cluster config. Unknown MPI implementations raise an error. +- **View names are globally unique**: view names must be unique across all environments in a recipe. +- **`mirrors.yaml` in recipes is unsupported**: use `--cache` CLI flag instead. +- **`default-view` must exist**: if set in `config.yaml`, the named view must be defined in `environments.yaml` (or be `modules`/`spack`). +- **`prefer` is auto-set**: if `null` in the recipe, Stackinator generates a `prefer` constraint using Spack's `%[when=...]` syntax to pin the default compiler. +- **Spack `uenv_tools` environment**: an internal environment named `uenv_tools` is injected into every build to install `squashfs`. Recipe authors must not use this name. diff --git a/stackinator/builder.py b/stackinator/builder.py index ae4997a1..011b7130 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -15,19 +15,12 @@ def install(src, dst, *, ignore=None, symlinks=False): - """Call shutil.copytree or shutil.copy2. copy2 is used if `src` is not a directory. - Afterwards run the equivalent of chmod a+rX dst.""" + """Call shutil.copytree or shutil.copy2, then apply chmod a+rX to dst.""" def apply_permissions_recursive(directory): - """Apply permissions recursively to an entire directory.""" - def set_permissions(path): - """Set permissions for a given path based on chmod a+rX equivalent.""" mode = os.stat(path).st_mode - # Always give read permissions for user, group, and others. new_mode = mode | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH - # If it's a directory or execute bit is set for owner or group, - # set execute bit for all. if stat.S_ISDIR(mode) or mode & (stat.S_IXUSR | stat.S_IXGRP): new_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH os.chmod(path, new_mode) @@ -40,15 +33,9 @@ def set_permissions(path): set_permissions(os.path.join(dirpath, filename)) if stat.S_ISDIR(os.stat(src).st_mode): - shutil.copytree( - src, - dst, - ignore=ignore, - symlinks=symlinks, - ) + shutil.copytree(src, dst, ignore=ignore, symlinks=symlinks) else: shutil.copy2(src, dst, follow_symlinks=symlinks) - # set permissions apply_permissions_recursive(dst) @@ -59,41 +46,28 @@ def __init__(self, args): if not path.is_absolute(): path = pathlib.Path.cwd() / path - # check that if the path exists that it is not a file if path.exists(): if not path.is_dir(): raise IOError("build path is not a directory") parts = path.parts - - # the build path can't be root if len(parts) == 1: raise IOError("build path can't be root '/'") - - # the build path can't be in /tmp because the build step rebinds /tmp. if parts[1] == "tmp": raise IOError("build path can't be in '/tmp'") - - # the build path can't be in $HOME because the build step rebinds $HOME - # NOTE that this would be much easier to determine with PosixPath.is_relative_to - # introduced in Python 3.9. home_parts = pathlib.Path.home().parts if (len(home_parts) <= len(parts)) and (home_parts == parts[: len(home_parts)]): raise IOError("build path can't be in '$HOME' or '~'") - # if path.is_relative_to(pathlib.Path.home()): - # raise IOError("build path can't be in '$HOME' or '~'") self.path = path self.root = pathlib.Path(__file__).parent.resolve() @property def configuration_meta(self): - """Meta data about the configuration and build""" return self._configuration_meta @configuration_meta.setter def configuration_meta(self, recipe): - # generate configuration meta data meta = {} meta["time"] = datetime.now().strftime("%Y%m%d %H:%M:%S") host_data = platform.uname() @@ -117,40 +91,10 @@ def configuration_meta(self, recipe): @property def environment_meta(self): - """The meta data file that describes the environments""" return self._environment_meta @environment_meta.setter def environment_meta(self, recipe): - """ - The output that we want to generate looks like the following, - Which should correspond directly to the environment_view_meta provided - by the recipe. - - { - name: "prgenv-gnu", - description: "useful programming tools", - mount: "/user-environment" - modules: { - "root": /user-environment/modules, - }, - views: { - "default": { - "root": /user-environment/env/default, - "activate": /user-environment/env/default/activate.sh, - "description": "simple devolpment env: compilers, MPI, python, cmake." - "env_vars": { - ... - } - }, - "tools": { - "root": /user-environment/env/tools, - "activate": /user-environment/env/tools/activate.sh, - "description": "handy tools" - } - } - } - """ conf = recipe.config meta = {} meta["name"] = conf["name"] @@ -165,7 +109,6 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): - # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" @@ -173,47 +116,34 @@ def generate(self, recipe): store_path.mkdir(exist_ok=True) tmp_path.mkdir(exist_ok=True) - # check out the version of spack - spack_version = recipe.spack_version - self._logger.debug(f"spack version for templates: {spack_version}") - - # set general build and configuration meta data for the project self.configuration_meta = recipe - - # set the environment view meta data self.environment_meta = recipe - # Clone the spack repository and check out commit if one was given + # Clone spack spack = recipe.config["spack"] - spack_repo = spack["repo"] - spack_commit = spack["commit"] spack_path = self.path / "spack" + spack_git_commit = self._git_clone("spack", spack["repo"], spack["commit"], spack_path) - spack_git_commit_result = self._git_clone("spack", spack_repo, spack_commit, spack_path) - - # Clone the spack-packages repository and check out commit if one was given + # Clone spack-packages spack_packages = spack["packages"] - spack_packages_repo = spack_packages["repo"] - spack_packages_commit = spack_packages["commit"] spack_packages_path = self.path / "spack-packages" - - spack_packages_git_commit_result = self._git_clone( + spack_packages_git_commit = self._git_clone( "spack-packages", - spack_packages_repo, - spack_packages_commit, + spack_packages["repo"], + spack_packages["commit"], spack_packages_path, ) spack_meta = { - "url": spack_repo, - "ref": spack_commit, - "commit": spack_git_commit_result, - "packages_url": spack_packages_repo, - "packages_ref": spack_packages_commit, - "packages_commit": spack_packages_git_commit_result, + "url": spack["repo"], + "ref": spack["commit"], + "commit": spack_git_commit, + "packages_url": spack_packages["repo"], + "packages_ref": spack_packages["commit"], + "packages_commit": spack_packages_git_commit, } - # load the jinja templating environment + # Jinja environment for templates template_path = self.root / "templates" jinja_env = jinja2.Environment( loader=jinja2.FileSystemLoader(template_path), @@ -221,29 +151,45 @@ def generate(self, recipe): lstrip_blocks=True, ) - # generate top level makefiles - makefile_template = jinja_env.get_template("Makefile") + # --- Write the unified spack.yaml --- + with (self.path / "spack.yaml").open("w") as f: + f.write(recipe.spack_yaml) + f.write("\n") + # --- Write packages.yaml (merged system + network + recipe packages) --- + with (self.path / "packages.yaml").open("w") as f: + f.write(yaml.dump(recipe.packages)) + + # --- Write config.yaml (install tree location) --- + config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}}} + with (self.path / "config.yaml").open("w") as f: + f.write(yaml.dump(config_yaml)) + + # --- Write Makefile --- + has_views = any(env_cfg["views"] for env_cfg in recipe.environments.values()) + makefile_template = jinja_env.get_template("Makefile") with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( cache=recipe.mirror, + push_to_cache=recipe.mirror is not None, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, - spack_version=spack_version, spack_meta=spack_meta, + environments=recipe.environments, + compiler_names=recipe.compiler_names, exclude_from_cache=["nvhpc", "cuda", "perl"], - verbose=False, + has_views=has_views, ) ) f.write("\n") + # --- Write Make.user --- make_user_template = jinja_env.get_template("Make.user") with (self.path / "Make.user").open("w") as f: f.write( make_user_template.render( - spack_version=spack_version, build_path=self.path, store=recipe.mount, no_bwrap=recipe.no_bwrap, @@ -252,11 +198,12 @@ def generate(self, recipe): ) f.write("\n") + # --- Copy static files from etc/ --- etc_path = self.root / "etc" - for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py"]: + for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py", "compiler-config.py"]: shutil.copy2(etc_path / f_etc, self.path / f_etc) - # used to configure both pre and post install hooks, if they are provided. + # --- Install hooks if provided --- hook_env = { "mount": recipe.mount, "config": recipe.mount / "config", @@ -264,263 +211,131 @@ def generate(self, recipe): "spack": self.path / "spack", } - # copy post install hook file, if provided - post_hook = recipe.post_install_hook - if post_hook is not None: - self._logger.debug("installing post-install-hook script") - jinja_recipe_env = jinja2.Environment(loader=jinja2.FileSystemLoader(recipe.path)) - post_hook_template = jinja_recipe_env.get_template("post-install") - post_hook_destination = store_path / "post-install-hook" - - with post_hook_destination.open("w") as f: - f.write(post_hook_template.render(env=hook_env, verbose=False)) - f.write("\n") - - os.chmod( - post_hook_destination, - os.stat(post_hook_destination).st_mode | stat.S_IEXEC, - ) - - # copy pre install hook file, if provided - pre_hook = recipe.pre_install_hook - if pre_hook is not None: - self._logger.debug("installing pre-install-hook script") - jinja_recipe_env = jinja2.Environment(loader=jinja2.FileSystemLoader(recipe.path)) - pre_hook_template = jinja_recipe_env.get_template("pre-install") - pre_hook_destination = store_path / "pre-install-hook" - - with pre_hook_destination.open("w") as f: - f.write(pre_hook_template.render(env=hook_env, verbose=False)) - f.write("\n") - - os.chmod( - pre_hook_destination, - os.stat(pre_hook_destination).st_mode | stat.S_IEXEC, - ) - - # Generate the system configuration: the compilers, environments, etc. - # that are defined for the target cluster. - config_path = self.path / "config" - config_path.mkdir(exist_ok=True) - packages_path = config_path / "packages.yaml" - - # the packages.yaml configuration that will be used when building all environments - # - the system packages.yaml with gcc removed - # - plus additional packages provided by the recipe - global_packages_yaml = yaml.dump(recipe.packages["global"]) - global_packages_path = config_path / "packages.yaml" - with global_packages_path.open("w") as fid: - fid.write(global_packages_yaml) - - # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: - dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) - - # Add custom spack package recipes, configured via Spack repos. - # Step 1: copy Spack repos to store_path where they will be used to - # build the stack, and then be part of the upstream provided - # to users of the stack. - # - # Packages in the recipe are prioritised over cluster specific packages, - # etc. The order of preference from highest to lowest is: - # - # 3. recipe/repo - # 2. cluster-config/repos.yaml - # - if the repos.yaml file exists it will contain a list of relative paths - # to search for package - # 1. builtin repo - - # Build a list of repos with packages to install. + for hook_name, hook_src in [("post-install", recipe.post_install_hook), ("pre-install", recipe.pre_install_hook)]: + if hook_src is not None: + self._logger.debug(f"installing {hook_name} script") + jinja_recipe_env = jinja2.Environment(loader=jinja2.FileSystemLoader(recipe.path)) + hook_template = jinja_recipe_env.get_template(hook_src.name) + hook_dst = store_path / f"{hook_name}-hook" + with hook_dst.open("w") as f: + f.write(hook_template.render(env=hook_env, verbose=False)) + f.write("\n") + os.chmod(hook_dst, os.stat(hook_dst).st_mode | stat.S_IEXEC) + + # --- Build the consolidated 'alps' spack package repo --- + # Precedence (highest first): recipe/repo > cluster repos.yaml entries > spack builtin repos = [] - - # check for a repo in the recipe if recipe.spack_repo is not None: self._logger.debug(f"adding recipe spack package repo: {recipe.spack_repo}") repos.append(recipe.spack_repo) - # look for repos.yaml file in the system configuration - repo_yaml = recipe.system_config_path / "repos.yaml" - if repo_yaml.exists() and repo_yaml.is_file(): - # open repos.yaml file and reat the list of repos - with repo_yaml.open() as fid: + repo_yaml_path = recipe.system_config_path / "repos.yaml" + if repo_yaml_path.exists(): + with repo_yaml_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) - P = raw["repos"] - - self._logger.debug(f"the system configuration has a repo file {repo_yaml} refers to {P}") - - # test each path - for rel_path in P: + for rel_path in raw["repos"]: repo_path = (recipe.system_config_path / rel_path).resolve() if spack_util.is_repo(repo_path): repos.append(repo_path) self._logger.debug(f"adding site spack package repo: {repo_path}") else: - self._logger.error(f"{repo_path} from {repo_yaml} is not a spack package repository") + self._logger.error(f"{repo_path} from {repo_yaml_path} is not a spack package repository") raise RuntimeError("invalid system-provided package repository") - self._logger.debug(f"full list of spack package repo: {repos}") - - # Delete the store/repo path, if it already exists. - # Do this so that incremental builds (though not officially supported) won't break if a repo is updated. repos_path = store_path / "repos" / "spack_repo" repo_dst = repos_path / "alps" - self._logger.debug(f"creating the stack spack repo in {repo_dst}") if repo_dst.exists(): - self._logger.debug(f"{repo_dst} exists ... deleting") shutil.rmtree(repo_dst) - - # create the repository step 1: create the repo directory pkg_dst = repo_dst / "packages" pkg_dst.mkdir(mode=0o755, parents=True) - self._logger.debug(f"created the repo packages path {pkg_dst}") - # create the repository step 2: create the repo.yaml file that - # configures alps and builtin repos with (repo_dst / "repo.yaml").open("w") as f: - f.write( - """\ -repo: - namespace: alps - api: v2.0 -""" - ) + f.write("repo:\n namespace: alps\n api: v2.0\n") + + # config/repos.yaml — consumed by SPACK_SYSTEM_CONFIG_PATH during the build + # and copied to store/config/repos.yaml by generate-config for downstream users + config_path = self.path / "config" + config_path.mkdir(exist_ok=True) - # create the repository step 2: create the repos.yaml file in build_path/config repos_yaml_template = jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: repo_path = recipe.mount / "repos" / "spack_repo" / "alps" builtin_repo_path = recipe.mount / "repos" / "spack_repo" / "builtin" - f.write( - repos_yaml_template.render( - repo_path=repo_path.as_posix(), - builtin_repo_path=builtin_repo_path.as_posix(), - verbose=False, - ) - ) + f.write(repos_yaml_template.render(repo_path=repo_path.as_posix(), builtin_repo_path=builtin_repo_path.as_posix())) f.write("\n") - # Iterate over the source repositories copying their contents to the consolidated repo in the uenv. - # Do overwrite packages that have been copied from an earlier source repo, enforcing a descending - # order of precidence. - if len(repos) > 0: - for repo_src in repos: - self._logger.debug(f"installing repo {repo_src}") - packages_path = repo_src / "packages" - for pkg_path in packages_path.iterdir(): - dst = pkg_dst / pkg_path.name - if pkg_path.is_dir() and not dst.exists(): - self._logger.debug(f" installing package {pkg_path} to {pkg_dst}") - install(pkg_path, dst) - elif dst.exists(): - self._logger.debug(f" NOT installing package {pkg_path}") - - # Copy the builtin repo to store, delete if it already exists. - spack_packages_builtin_path = spack_packages_path / "repos" / "spack_repo" / "builtin" - spack_packages_store_path = store_path / "repos" / "spack_repo" / "builtin" - self._logger.debug(f"copying builtin repo from {spack_packages_builtin_path} to {spack_packages_store_path}") - if spack_packages_store_path.exists(): - self._logger.debug(f"{spack_packages_store_path} exists ... deleting") - shutil.rmtree(spack_packages_store_path) - install(spack_packages_builtin_path, spack_packages_store_path) - - # Generate the makefile and spack.yaml files that describe the compilers - compiler_files = recipe.compiler_files - compiler_path = self.path / "compilers" - compiler_path.mkdir(exist_ok=True) - with (compiler_path / "Makefile").open(mode="w") as f: - f.write(compiler_files["makefile"]) - - for name, files in compiler_files["config"].items(): - compiler_config_path = compiler_path / name - compiler_config_path.mkdir(exist_ok=True) - for file, raw in files.items(): - with (compiler_config_path / file).open(mode="w") as f: - f.write(raw) - - # generate the makefile and spack.yaml files that describe the environments - environment_files = recipe.environment_files - environments_path = self.path / "environments" - os.makedirs(environments_path, exist_ok=True) - with (environments_path / "Makefile").open(mode="w") as f: - f.write(environment_files["makefile"]) - - for name, yml in environment_files["config"].items(): - env_config_path = environments_path / name - env_config_path.mkdir(exist_ok=True) - with (env_config_path / "spack.yaml").open(mode="w") as f: - f.write(yml) - - # generate the makefile that generates the configuration for the spack - # installation in the generate-config sub-directory of the build path. - make_config_template = jinja_env.get_template("Makefile.generate-config") + if recipe.mirror: + with (config_path / "mirrors.yaml").open("w") as fid: + fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + + # Copy package definitions into the alps repo (recipe > site > nothing) + for repo_src in repos: + for pkg_path in (repo_src / "packages").iterdir(): + dst = pkg_dst / pkg_path.name + if pkg_path.is_dir() and not dst.exists(): + install(pkg_path, dst) + + # Copy builtin repo from spack-packages + spack_builtin_src = spack_packages_path / "repos" / "spack_repo" / "builtin" + spack_builtin_dst = store_path / "repos" / "spack_repo" / "builtin" + if spack_builtin_dst.exists(): + shutil.rmtree(spack_builtin_dst) + install(spack_builtin_src, spack_builtin_dst) + + # --- generate-config subdirectory --- generate_config_path = self.path / "generate-config" generate_config_path.mkdir(exist_ok=True) - # write generate-config/Makefile - all_compilers = [x for x in recipe.compilers.keys()] + make_config_template = jinja_env.get_template("Makefile.generate-config") with (generate_config_path / "Makefile").open("w") as f: f.write( make_config_template.render( modules=recipe.with_modules, build_path=self.path.as_posix(), - all_compilers=all_compilers, - release_compilers=all_compilers, - verbose=False, + compiler_names=recipe.compiler_names, ) ) + f.write("\n") - # write modules/modules.yaml + # --- modules --- if recipe.with_modules: - generate_modules_path = self.path / "modules" - generate_modules_path.mkdir(exist_ok=True) - with (generate_modules_path / "modules.yaml").open("w") as f: + modules_path = self.path / "modules" + modules_path.mkdir(exist_ok=True) + with (modules_path / "modules.yaml").open("w") as f: yaml.dump(recipe.modules, f) - # write the meta data + # --- metadata --- meta_path = store_path / "meta" meta_path.mkdir(exist_ok=True) - # write a json file with basic meta data + with (meta_path / "configure.json").open("w") as f: - # default serialisation is str to serialise the pathlib.PosixPath f.write(json.dumps(self.configuration_meta, sort_keys=True, indent=2, default=str)) f.write("\n") - # write a json file with the environment view meta data with (meta_path / "env.json.in").open("w") as f: - # default serialisation is str to serialise the pathlib.PosixPath f.write(json.dumps(self.environment_meta, sort_keys=True, indent=2, default=str)) f.write("\n") - # copy the recipe to a recipe subdirectory of the meta path meta_recipe_path = meta_path / "recipe" - meta_recipe_path.mkdir(exist_ok=True) if meta_recipe_path.exists(): shutil.rmtree(meta_recipe_path) install(recipe.path, meta_recipe_path, ignore=shutil.ignore_patterns(".git")) - # create the meta/extra path and copy recipe meta data if it exists meta_extra_path = meta_path / "extra" - meta_extra_path.mkdir(exist_ok=True) if meta_extra_path.exists(): shutil.rmtree(meta_extra_path) + meta_extra_path.mkdir(exist_ok=True) if recipe.user_extra is not None: - self._logger.debug(f"copying extra recipe meta data to {meta_extra_path}") install(recipe.user_extra, meta_extra_path) - # create debug helper script - debug_script_path = self.path / "stack-debug.sh" - debug_script_template = jinja_env.get_template("stack-debug.sh") - with debug_script_path.open("w") as f: + # --- debug helper --- + debug_template = jinja_env.get_template("stack-debug.sh") + with (self.path / "stack-debug.sh").open("w") as f: f.write( - debug_script_template.render( + debug_template.render( mount_path=recipe.mount, build_path=str(self.path), use_bwrap=not recipe.no_bwrap, - spack_version=spack_version, - verbose=False, ) ) f.write("\n") @@ -528,8 +343,6 @@ def generate(self, recipe): def _git_clone(self, name, repo, commit, path): if not (path / ".git").is_dir(): self._logger.info(f"{name}: clone repository {repo} to {path}") - - # clone the repository capture = subprocess.run( ["git", "clone", "--filter=tree:0", repo, path], shell=False, @@ -537,7 +350,6 @@ def _git_clone(self, name, repo, commit, path): stderr=subprocess.STDOUT, ) self._logger.debug(capture.stdout.decode("utf-8")) - if capture.returncode != 0: self._logger.error(f"error cloning the repository {repo}") capture.check_returncode() @@ -545,7 +357,6 @@ def _git_clone(self, name, repo, commit, path): self._logger.info(f"{name}: {repo} already cloned to {path}") if commit: - # Fetch the specific branch self._logger.info(f"{name}: fetching {commit}") capture = subprocess.run( ["git", "-C", path, "fetch", "origin", commit], @@ -554,12 +365,9 @@ def _git_clone(self, name, repo, commit, path): stderr=subprocess.STDOUT, ) self._logger.debug(capture.stdout.decode("utf-8")) - if capture.returncode != 0: - self._logger.debug(f"unable to fetch {commit}") capture.check_returncode() - # Check out the specific branch self._logger.info(f"{name}: checking out {commit}") capture = subprocess.run( ["git", "-C", path, "checkout", commit], @@ -568,15 +376,12 @@ def _git_clone(self, name, repo, commit, path): stderr=subprocess.STDOUT, ) self._logger.debug(capture.stdout.decode("utf-8")) - if capture.returncode != 0: - self._logger.debug(f"unable to change to the requested commit {commit}") capture.check_returncode() else: self._logger.info(f"{name}: no commit set") - # get the commit - git_commit_result = ( + git_commit = ( subprocess.run( ["git", "-C", path, "rev-parse", "HEAD"], shell=False, @@ -586,7 +391,5 @@ def _git_clone(self, name, repo, commit, path): .stdout.strip() .decode("utf-8") ) - - self._logger.info(f"{name}: commit hash is {git_commit_result}") - - return git_commit_result + self._logger.info(f"{name}: commit hash is {git_commit}") + return git_commit diff --git a/stackinator/etc/Make.inc b/stackinator/etc/Make.inc index cf05543d..04a2789f 100644 --- a/stackinator/etc/Make.inc +++ b/stackinator/etc/Make.inc @@ -1,8 +1,7 @@ # vi: filetype=make SPACK ?= spack - -SPACK_ENV = $(SPACK) -e $(dir $@) +SPACK_HELPER := $(SPACK) --color=never ifndef STORE $(error STORE should point to a Spack install root) @@ -14,19 +13,3 @@ endif store: mkdir -p $(STORE) - -# Concretization -%/spack.lock: %/spack.yaml %/config.yaml %/packages.yaml - /usr/bin/time -f '"%C" took %e seconds.' $(SPACK_ENV) concretize -f - -# Generate Makefiles for the environment install -%/Makefile: %/spack.lock - $(SPACK_ENV) env depfile --make-target-prefix $*/generated -o $@ - -# For generating {compilers,config,packages}.yaml files. -%.yaml: export SPACK_USER_CONFIG_PATH=$(abspath $(dir $@)) -%.yaml: - touch $@ - -# Because Spack doesn't know how to find compilers, we help it by getting the bin folder of gcc, clang, nvc given a install prefix -compiler_bin_dirs = $$(find $(1) '(' -name gcc -o -name clang -o -name nvc -o -name icx ')' -path '*/bin/*' '(' -type f -o -type l ')' -exec dirname {} +) diff --git a/stackinator/etc/compiler-config.py b/stackinator/etc/compiler-config.py new file mode 100644 index 00000000..f1d457bf --- /dev/null +++ b/stackinator/etc/compiler-config.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +""" +Generate or update a packages.yaml file with compiler externals derived from +installed Spack packages. Intended to be run as: + + spack -e BUILD_ROOT python compiler-config.py OUTPUT_YAML COMPILER [COMPILER ...] + +If OUTPUT_YAML already exists (e.g. the build packages.yaml), the compiler +entries are merged in rather than replacing existing content. +""" + +import argparse +import os +import sys + +import yaml + + +_COMPILER_BINS = { + "gcc": [("gcc", "c"), ("g++", "cxx"), ("gfortran", "fortran")], + "llvm": [("clang", "c"), ("clang++", "cxx"), ("flang-new", "fortran")], + "llvm-amdgpu": [("clang", "c"), ("clang++", "cxx"), ("flang-new", "fortran")], + "nvhpc": [("nvc", "c"), ("nvc++", "cxx"), ("nvfortran", "fortran")], + "intel-oneapi-compilers": [("icx", "c"), ("icpx", "cxx"), ("ifx", "fortran")], +} + + +def find_compiler_bins(prefix, compiler_name): + """ + Return a dict mapping language keys (c, cxx, fortran) to absolute binary + paths found under prefix, or None if nothing was found. + """ + candidates = _COMPILER_BINS.get(compiler_name, []) + result = {} + for root, _dirs, files in os.walk(prefix): + for exe, lang in candidates: + if lang not in result and exe in files: + full = os.path.join(root, exe) + if os.access(full, os.X_OK): + result[lang] = full + if len(result) == len(candidates): + break + return result or None + + +def build_compiler_packages(compiler_names): + """ + Query the active Spack DB for each compiler name and return a dict + suitable for merging into packages.yaml. + """ + import spack.store + + packages = {} + for name in compiler_names: + specs = list(spack.store.STORE.db.query(name, explicit=False)) + if not specs: + print(f" compiler-config: no installed specs found for '{name}'", file=sys.stderr) + continue + + externals = [] + for spec in specs: + prefix = str(spec.prefix) + bins = find_compiler_bins(prefix, name) + if not bins: + print(f" compiler-config: no binaries found for {name} at {prefix}", file=sys.stderr) + continue + externals.append( + { + "spec": f"{spec.name}@{spec.version}", + "prefix": prefix, + "extra_attributes": {"compilers": bins}, + } + ) + print(f" compiler-config: found {name}@{spec.version} at {prefix}", file=sys.stderr) + + if externals: + packages[name] = {"externals": externals, "buildable": False} + + return packages + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", help="Path to packages.yaml to create or update") + parser.add_argument("compilers", nargs="+", help="Compiler package names to query") + args = parser.parse_args() + + # Load existing content if the file already exists (merge mode). + existing = {} + if os.path.isfile(args.output): + with open(args.output) as fid: + existing = yaml.safe_load(fid) or {} + + compiler_packages = build_compiler_packages(args.compilers) + + # Merge: compiler entries overwrite any existing entry for the same package name. + merged = existing.copy() + merged.setdefault("packages", {}).update(compiler_packages) + + with open(args.output, "w") as fid: + yaml.dump(merged, fid, default_flow_style=False) + + print(f" compiler-config: wrote {args.output}", file=sys.stderr) + + +main() diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 30bf8b58..6601b414 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -78,15 +78,6 @@ def __init__(self, args): "tcl", (self.mount / "modules").as_posix() ) - # DEPRECATED field `config:modules` - if "modules" in self.config: - self._logger.warning("boolean field config.yaml:modules has been deprecated") - - if self.with_modules != self.config["modules"]: - self._logger.error(f"config.yaml:modules:{self.config['modules']}") - self._logger.error(f"modules.yaml:{self.with_modules}") - raise RuntimeError("conflicting modules configuration detected") - self._logger.debug("creating packages") # load recipe/packages.yaml -> recipe_packages (if it exists) @@ -101,22 +92,14 @@ def __init__(self, args): system_packages = {} system_packages_path = self.system_config_path / "packages.yaml" if system_packages_path.is_file(): - # load system yaml with system_packages_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) system_packages = raw["packages"] - # extract gcc packages from system packages - # remove gcc from packages afterwards - if "gcc" in system_packages: - gcc_packages = {"gcc": system_packages["gcc"]} - del system_packages["gcc"] - else: + if "gcc" not in system_packages: raise RuntimeError("The system packages.yaml file does not provide gcc") - # load the optional network.yaml from system config: - # - meta data about mpi - # - package information for network libraries (libfabric, openmpi, cray-mpich, ... etc) + # load the optional network.yaml from system config network_path = self.system_config_path / "network.yaml" network_packages = {} mpi_templates = {} @@ -132,12 +115,9 @@ def __init__(self, args): # note that the order that package sets are specified in is significant. # arguments to the right have higher precedence. - self.packages = { - # the package definition used in every environment - "global": {"packages": system_packages | network_packages | recipe_packages}, - # the package definition used to build gcc (requires system gcc to bootstrap) - "gcc": {"packages": system_packages | gcc_packages | recipe_packages}, - } + # Global packages.yaml: system + network + recipe packages. + # gcc is included here (unlike v2 which isolated it for the bootstrap step). + self.packages = {"packages": system_packages | network_packages | recipe_packages} # required environments.yaml file environments_path = self.path / "environments.yaml" @@ -147,42 +127,26 @@ def __init__(self, args): with environments_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) - # add a special environment that installs tools required later in the build process. - # currently we only need squashfs for creating the squashfs file. - raw["uenv_tools"] = { - "compiler": ["gcc"], - "network": {"mpi": None, "specs": None}, - "unify": True, - "duplicates": {"strategy": "minimal"}, - "deprecated": False, - "specs": ["squashfs"], - "views": {}, - } schema.EnvironmentsValidator.validate(raw) + self._check_environments_v3(raw) self.generate_environment_specs(raw) # check that the default view exists (if one has been set) self._default_view = self.config["default-view"] if self._default_view is not None: available_views = [view["name"] for env in self.environments.values() for view in env["views"]] - # add the modules and spack views to the list of available views if self.with_modules: available_views.append("modules") available_views.append("spack") if self._default_view not in available_views: - self._logger.error( - f"The default-view {self._default_view} is not the name of a view in the environments.yaml " - "definition (one of {[name for name in available_views]}" + raise RuntimeError( + f"The default-view '{self._default_view}' is not the name of a view defined in environments.yaml" ) - raise RuntimeError("Ivalid default-view in the recipe.") - # optional mirror configurtion + # mirrors.yaml in a recipe is no longer supported — use --cache instead mirrors_path = self.path / "mirrors.yaml" if mirrors_path.is_file(): - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") + raise RuntimeError("mirrors.yaml in a recipe is not supported; use the --cache flag instead.") self.mirror = (args.cache, self.mount) @@ -198,10 +162,14 @@ def __init__(self, args): else: self._logger.debug("no pre install hook provided") - # determine the version of spack being used: - # currently this just returns 1.0... develop is ignored - # --develop flag will imply the next release of spack after 1.0 is supported properly - self.spack_version = self.find_spack_version(args.develop) + def _check_environments_v3(self, raw): + """Warn about fields that are ignored in v3 recipes.""" + for name, config in raw.items(): + if config and config.get("packages"): + self._logger.warning( + f"environment '{name}': the 'packages' field is ignored in v3 recipes. " + "Add external packages to the recipe's packages.yaml instead." + ) # Returns: # Path: if the recipe contains a spack package repository @@ -288,11 +256,6 @@ def config(self, config_path): def with_modules(self) -> bool: return self.modules is not None - # In Stackinator 6 we replaced logic required to determine the - # pre 1.0 Spack version. - def find_spack_version(self, develop): - return "1.0" - @property def default_view(self): return self._default_view @@ -312,83 +275,68 @@ def environment_view_meta(self): "mount": str(self.mount), "view_path": str(view["config"]["root"]), } - env = EnvVarSet.from_envvars(view["extra"]["env_vars"], substitutions) + env_vars = EnvVarSet.from_envvars(view["extra"]["env_vars"], substitutions) except Exception as err: raise RuntimeError(f'In view "{view["name"]}": {err}') view_meta[view["name"]] = { "root": view["config"]["root"], - "description": "", # leave the description empty for now - "recipe_variables": env.as_dict(), + "description": "", + "recipe_variables": env_vars.as_dict(), } return view_meta + @property + def compiler_names(self): + """Names of the compiler packages installed in this recipe.""" + return list(self.compilers.keys()) + # creates the self.environments field that describes the full specifications # for all of the environments sets, grouped in environments, from the raw # environments.yaml input. def generate_environment_specs(self, raw): environments = raw - # enumerate large binary packages that should not be pushed to binary caches for _, config in environments.items(): config["exclude_from_cache"] = ["cuda", "nvhpc", "perl"] - # check the environment descriptions and ammend where features are missing for name, config in environments.items(): if ("specs" not in config) or (config["specs"] is None): environments[name]["specs"] = [] - # Complete configuration of MPI in each environment - # this involves generate specs for the chosen MPI implementation - # and (optionally) additional dependencies like libfabric, which are - # appended to the list of specs in the environment. + # Resolve MPI specs from the network field for name, config in environments.items(): - # the "mpi" entry records the name of the MPI implementation used by the environment. - # set it to none by default, and have it set if the config["network"] description specifies - # an MPI implementation. environments[name]["mpi"] = None if config["network"]: - # we will build a list of additional specs related to MPI, libfabric, etc to add to the list of specs - # in the generated spack.yaml file. - # start with an empty list: specs = [] if config["network"]["mpi"] is not None: spec = config["network"]["mpi"].strip() - # find the name of the MPI package match = re.match(r"^([A-Za-z][A-Za-z0-9_-]*)", spec) if match: mpi_name = match.group(1) - supported_mpis = [k for k in self.mpi_templates.keys()] + supported_mpis = list(self.mpi_templates.keys()) if mpi_name not in supported_mpis: - raise Exception(f"{mpi_name} is not a supported MPI version: try one of {supported_mpis}.") + raise Exception( + f"{mpi_name} is not a supported MPI: try one of {supported_mpis}." + ) else: raise Exception(f"{spec} is not a valid MPI spec") - # add the mpi spec to the list of explicit specs specs.append(spec) - # if the recipe provided explicit specs for dependencies, inject them: if config["network"]["specs"]: specs += config["network"]["specs"] - # otherwise inject dependencies from network.yaml (if they exist) elif self.mpi_templates[mpi_name]["specs"]: specs += self.mpi_templates[mpi_name]["specs"] environments[name]["mpi"] = mpi_name environments[name]["specs"] += specs - # set constraints that ensure the the main compiler is always used to build packages - # that do not explicitly request a compiler. + # Auto-generate a prefer constraint that pins the default compiler for name, config in environments.items(): - # if the recipe provided no "prefer" settings, provide a default one that - # nudges Spack towards using the first compiler (we don't think that this actually - # has much effect). - # With this set, the user can the customise the compiler to use as on a package spec, e.g. - # hdf5+mpi+fortran %fortran=nvhpc - # Which will compile the upstream MPI with nvfortran, as well as downstream dependendencies. if config["prefer"] is None: compiler = config["compiler"][0] # spack uses a different name for the intel oneapi compilers @@ -399,20 +347,20 @@ def generate_environment_specs(self, raw): f"%[when=%c] c={compiler} %[when=%cxx] cxx={compiler} %[when=%fortran] fortran={compiler}" ] - # Create all meta data for all of the views. + # Build view metadata env_names = set() for name, config in environments.items(): views = [] for view_name, vc in config["views"].items(): if view_name in env_names: - raise Exception(f"An environment view with the name '{name}' already exists.") + raise Exception(f"A view named '{view_name}' is defined more than once.") env_names.add(view_name) view_config = copy.deepcopy(vc) # set some default values: - # ["link"] = "roots" - # ["uenv"]["add_compilers"] = True - # ["uenv"]["prefix_paths"] = {} - # ["uenv"]["env_vars"] = {"set": [], "unset": [], "prepend_path": [], "append_path": []} + # view_config["link"] = "roots" + # view_config["uenv"]["add_compilers"] = True + # view_config["uenv"]["prefix_paths"] = {} + # view_config["uenv"]["env_vars"] = {"set": [], "unset": [], "prepend_path": [], "append_path": []} if view_config is None: view_config = {} view_config.setdefault("link", "roots") @@ -429,8 +377,14 @@ def generate_environment_specs(self, raw): [f"{pname}={':'.join(paths)}" for pname, paths in view_config["uenv"]["prefix_paths"].items()] ) view_config["uenv"]["prefix_string"] = prefix_string + # note: root is stored as a string (not pathlib.PosixPath) to avoid + # serialisation issues when the config is written to spack.yaml. view_config["root"] = str(self.mount / "env" / view_name) + # The "uenv" field is stackinator-specific metadata (compiler paths, + # env-var rules) — not a spack view config field. Pop it before + # passing view_config to spack; it travels separately as "extra" and + # is consumed by envvars.py during the view-generation step. extra = view_config.pop("uenv") views.append({"name": view_name, "config": view_config, "extra": extra}) @@ -443,52 +397,18 @@ def generate_environment_specs(self, raw): def generate_compiler_specs(self, raw): compilers = {} - cache_exclude = ["cuda", "nvhpc", "perl"] - gcc = {} - # gcc["packages"] = { - # "external": [ "perl", "m4", "autoconf", "automake", "libtool", "gawk", "python", "texinfo", "gawk", ], - # } gcc_version = raw["gcc"]["version"] - gcc["specs"] = [f"gcc@{gcc_version} + bootstrap"] - gcc["exclude_from_cache"] = [] - - compilers["gcc"] = gcc - - if raw["nvhpc"] is not None: - nvhpc = {} - nvhpc_version = raw["nvhpc"]["version"] - nvhpc["packages"] = False - nvhpc["specs"] = [f"nvhpc@{nvhpc_version} ~mpi~blas~lapack"] - - nvhpc["exclude_from_cache"] = cache_exclude - compilers["nvhpc"] = nvhpc - - if raw["llvm"] is not None: - llvm = {} - llvm_version = raw["llvm"]["version"] - llvm["packages"] = False - llvm["specs"] = [f"llvm@{llvm_version} +clang ~gold"] - - llvm["exclude_from_cache"] = cache_exclude - compilers["llvm"] = llvm - - if raw["llvm-amdgpu"] is not None: - llvm_amdgpu = {} - llvm_amdgpu_version = raw["llvm-amdgpu"]["version"] - llvm_amdgpu["packages"] = False - llvm_amdgpu["specs"] = [f"llvm-amdgpu@{llvm_amdgpu_version}"] - - llvm_amdgpu["exclude_from_cache"] = cache_exclude - compilers["llvm-amdgpu"] = llvm_amdgpu - - if raw["intel-oneapi-compilers"] is not None: - oneapi = {} - oneapi_version = raw["intel-oneapi-compilers"]["version"] - oneapi["packages"] = False - oneapi["specs"] = [f"intel-oneapi-compilers@{oneapi_version}"] - - oneapi["exclude_from_cache"] = cache_exclude - compilers["intel-oneapi-compilers"] = oneapi + compilers["gcc"] = {"specs": [f"gcc@{gcc_version} +bootstrap"]} + + for name, spec_template in [ + ("nvhpc", "nvhpc@{version} ~mpi~blas~lapack"), + ("llvm", "llvm@{version} +clang ~gold"), + ("llvm-amdgpu", "llvm-amdgpu@{version}"), + ("intel-oneapi-compilers", "intel-oneapi-compilers@{version}"), + ]: + if raw.get(name) is not None: + version = raw[name]["version"] + compilers[name] = {"specs": [spec_template.format(version=version)]} self.compilers = compilers @@ -513,58 +433,21 @@ def mount(self): return pathlib.Path(self.config["store"]) @property - def compiler_files(self): - files = {} - + def spack_yaml(self): + """Render the unified spack.yaml for this recipe.""" env = jinja2.Environment( loader=jinja2.FileSystemLoader(self.template_path), trim_blocks=True, lstrip_blocks=True, ) + env.filters["py2yaml"] = schema.py2yaml - makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None - files["makefile"] = makefile_template.render( - compilers=self.compilers, - push_to_cache=push_to_cache, - spack_version=self.spack_version, - ) - - files["config"] = {} - for compiler, config in self.compilers.items(): - spack_yaml_template = env.get_template(f"compilers.{compiler}.spack.yaml") - files["config"][compiler] = {} - # compilers//spack.yaml - files["config"][compiler]["spack.yaml"] = spack_yaml_template.render(config=config) - # compilers/gcc/packages.yaml - if compiler == "gcc": - files["config"][compiler]["packages.yaml"] = yaml.dump(self.packages["gcc"]) + has_views = any(env_cfg["views"] for env_cfg in self.environments.values()) - return files - - @property - def environment_files(self): - files = {} - - jenv = jinja2.Environment( - loader=jinja2.FileSystemLoader(self.template_path), - trim_blocks=True, - lstrip_blocks=True, - ) - jenv.filters["py2yaml"] = schema.py2yaml - - makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.mirror is not None - files["makefile"] = makefile_template.render( + template = env.get_template("spack.yaml") + return template.render( + compilers=self.compilers, environments=self.environments, - push_to_cache=push_to_cache, - spack_version=self.spack_version, + store=self.mount, + has_views=has_views, ) - - files["config"] = {} - for env, config in self.environments.items(): - spack_yaml_template = jenv.get_template("environments.spack.yaml") - # generate the spack.yaml file - files["config"][env] = spack_yaml_template.render(config=config, name=env, store=self.mount) - - return files diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a9842..d75a99bc 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -80,19 +80,18 @@ def validate(self, instance: dict): def check_config_version(instance): rversion = instance.get("version", 1) - if rversion != 2: - if rversion == 1: + if rversion != 3: + if rversion in (1, 2): root_logger.error( - dedent(""" - The recipe is an old version 1 recipe for Spack v0.23 and earlier. - This version of Stackinator supports Spack 1.0, and has deprecated support for Spack v0.23. - Use version 5 of stackinator, which can be accessed via the releases/v5 branch: - git switch releases/v5 + dedent(f""" + The recipe uses version {rversion} of the uenv recipe format. + Stackinator v7 only supports version 3 recipes (Spack 1.2+). - If this recipe is to be used with Spack 1.0, then please add the field 'version: 2' to - config.yaml in your recipe. + To build version {rversion} recipes, use Stackinator v6: + git switch releases/v6 - For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration + To port this recipe to version 3, see: + https://eth-cscs.github.io/stackinator/porting/ """) ) raise RuntimeError("incompatible uenv recipe version") @@ -100,7 +99,7 @@ def check_config_version(instance): root_logger.error( dedent(f""" The config.yaml file sets an unknown recipe version={rversion}. - This version of Stackinator supports version 2 recipes. + Stackinator v7 supports version 3 recipes. For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration """) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index d9de503e..888d33c6 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -67,7 +67,7 @@ "type": "number", "default": 1, "minimum": 1, - "maximum": 2 + "maximum": 3 } } } diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea58..75ca36ab 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -1,17 +1,14 @@ {% set pipejoiner = joiner('|') %} -include Make.user -.PHONY: compilers environments generate-config clean +.PHONY: all generate-config clean -all: environments +all: store.squashfs -# Keep track of what Spack version was used. spack-version: $(SANDBOX) $(SPACK) --version > $@ -# Do some sanity checks: (a) are we not on cray, (b) are we using the same -# version as before, (c) ensure that the concretizer is bootstrapped to avoid a -# race where multiple processes start doing that. +# Sanity check: confirm spack works and bootstrap the concretizer. spack-setup: spack-version @printf "spack arch... " ; \ arch="$$($(SANDBOX) $(SPACK) arch)"; \ @@ -28,79 +25,105 @@ spack-setup: spack-version printf " success\n"; \ touch spack-setup +{% if pre_install_hook %} pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook + touch pre-install +{% endif %} mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} - $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} +{% if cache %} + $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) buildcache keys --install --trust +{% if cache.key %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} - {% endif %} +{% endif %} +{% endif %} touch mirror-setup -compilers: mirror-setup - $(SANDBOX) $(MAKE) -C $@ +concretize: mirror-setup + $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) concretize + touch concretize + +install: concretize + $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) install + touch install -generate-config: compilers - $(SANDBOX) $(MAKE) -C $@ +{% if push_to_cache and cache.key %} +cache-push: install + $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) buildcache create --rebuild-index --only=package alpscache \ + $$($(SANDBOX) $(SPACK_HELPER) -e $(BUILD_ROOT) find --format '{name};{/hash};version={version}' \ + | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ + | grep -v -E 'version=git\.'\ + | cut -d ';' -f2)\ + | grep -Ev '^==> Fetching|^gpg:' + touch cache-push +{% endif %} -environments: compilers - $(SANDBOX) $(MAKE) -C $@ +# Generate compiler-config.yaml for use by view and generate-config steps. +compiler-config.yaml: install + $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) python $(BUILD_ROOT)/compiler-config.py \ + $(BUILD_ROOT)/compiler-config.yaml \ + {{ compiler_names | join(' ') }} + +# Generate activate.sh and env.json for each environment view. +{% for name, config in environments.items() %} +{% for view in config.views %} +views/{{ view.name }}: install compiler-config.yaml + mkdir -p $(dir $@) + $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) env activate --with-view {{ view.name }} --sh \ + > $(STORE)/env/{{ view.name }}/activate.sh + $(SANDBOX) $(BUILD_ROOT)/envvars.py view \ + {% if view.extra.add_compilers %}--compilers=$(BUILD_ROOT)/compiler-config.yaml {% endif %}\ + --prefix_paths="{{ view.extra.prefix_string }}" \ + $(STORE)/env/{{ view.name }} \ + $(BUILD_ROOT) + touch $@ + +{% endfor %} +{% endfor %} +views: {% for name, config in environments.items() %}{% for view in config.views %}views/{{ view.name }} {% endfor %}{% endfor %} + + touch views + +generate-config: install compiler-config.yaml + $(SANDBOX) $(MAKE) -C generate-config {% if modules %} -modules-done: environments generate-config - $(SANDBOX) $(SPACK) -C $(BUILD_ROOT)/modules module tcl refresh --upstream-modules --delete-tree --yes-to-all +modules-done: install generate-config + $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) module tcl refresh --upstream-modules --delete-tree --yes-to-all touch modules-done {% endif %} +env-meta: generate-config views{% if modules %} modules-done{% endif %} -env-meta: generate-config environments{% if modules %} modules-done{% endif %} - - $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}' --spack-packages='{{ spack_meta.packages_url }},{{ spack_meta.packages_ref }},{{ spack_meta.packages_commit }}' $(STORE) + $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv \ + {% if modules %}--modules {% endif %}\ + --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}' \ + --spack-packages='{{ spack_meta.packages_url }},{{ spack_meta.packages_ref }},{{ spack_meta.packages_commit }}' \ + $(STORE) touch env-meta +{% if post_install_hook %} post-install: env-meta - {% if post_install_hook %} $(SANDBOX) $(STORE)/post-install-hook - {% endif %} touch post-install +{% endif %} + +store.squashfs: env-meta{% if post_install_hook %} post-install{% endif %}{% if push_to_cache and cache.key %} cache-push{% endif %} -# Create a squashfs file from the installed software. -store.squashfs: post-install - # clean up the __pycache__ paths in the repo $(SANDBOX) find $(STORE)/repos -type d -name __pycache__ -exec rm -r {} + $(SANDBOX) chmod -R a+rX $(STORE) - $(SANDBOX) env -u SOURCE_DATE_EPOCH "$$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" $(STORE) $@ -force-uid nobody -force-gid nobody -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3 - -# Force push all built packages to the build cache -cache-force: mirror-setup -{% if cache.key %} - $(warning ================================================================================) - $(warning Generate the config in order to force push partially built compiler environments) - $(warning if this step is performed with partially built compiler envs, you will) - $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) - $(warning ================================================================================) - $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package alpscache \ - $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ - | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ - | grep -v -E 'version=git\.'\ - | cut -d ';' -f2)\ - | grep -Ev '^==> Fetching|^gpg:' -{% else %} - $(warning "pushing to the build cache is not enabled. See the documentation on how to add a key: https://eth-cscs.github.io/stackinator/build-caches/") -{% endif %} - -# A backup of all the generated files during the build, useful for posterity, -# excluding the binaries themselves, since they're in the squashfs file -build.tar.gz: spack-version Make.user Make.inc Makefile | environments - tar czf $@ $^ $$(find environments compilers config -maxdepth 2 -name Makefile -o -name '*.yaml') + $(SANDBOX) env -u SOURCE_DATE_EPOCH \ + "$$($(SANDBOX) $(SPACK_HELPER) -e $(BUILD_ROOT) find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" \ + $(STORE) $@ -force-uid nobody -force-gid nobody \ + -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3 -# Clean generate files, does *not* remove installed software. clean: - rm -rf -- $(wildcard */*/spack.lock) $(wildcard */*/.spack-env) $(wildcard */*/Makefile) $(wildcard */*/generated) $(wildcard cache) $(wildcard compilers/*/config.yaml) $(wildcard compilers/*/packages.yaml) $(wildcard environments/*/config.yaml) $(wildcard environments/*/packages.yaml) post-install modules-done env-meta store.squashfs + rm -rf -- spack-version spack-setup{% if pre_install_hook %} pre-install{% endif %} mirror-setup \ + concretize install{% if push_to_cache and cache.key %} cache-push{% endif %} \ + compiler-config.yaml views generate-config/.done \ + {% if modules %}modules-done {% endif %}env-meta{% if post_install_hook %} post-install{% endif %} \ + store.squashfs spack-bootstrap-output include Make.inc diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers deleted file mode 100644 index 8418fbf0..00000000 --- a/stackinator/templates/Makefile.compilers +++ /dev/null @@ -1,99 +0,0 @@ -{% set pipejoiner = joiner('|') %} --include ../Make.user - -MAKEFLAGS += --output-sync=recurse - -.PHONY: all .locks .packages.yaml - -all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfor %} - - -# Ensure that spack.lock files are never removed as intermediate files... -.locks:{% for compiler in compilers %} {{ compiler }}/spack.lock{% endfor %} - - -# Ensure that package yaml files are never removed as intermediate files... -.packages.yaml:{% for compiler in compilers %} {{ compiler }}/packages.yaml{% endfor %} - - -{% for compiler, config in compilers.items() %} -{{ compiler }}/generated/build_cache: {{ compiler }}/generated/env -{% if push_to_cache %} - $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package alpscache \ - $$($(SPACK_HELPER) -e ./{{ compiler }} find --format '{name};{/hash}' \ - | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ - | cut -d ';' -f2) -{% endif %} - touch $@ - -{% endfor %} - -# Configure the install location. -{% for compiler in compilers %}{{ compiler }}/config.yaml {% endfor %}: | store - $(SPACK) config --scope=user add config:install_tree:root:$(STORE) - -# Configure external system dependencies for each compiler toolchain -{% for compiler, config in compilers.items() %} -{% if config.packages and config.packages.external %} -{{ compiler }}/packages.yaml: - $(SPACK) external find --scope=user {% for package in config.packages.external %} {{package}}{% endfor %} - -{% endif %} -{% endfor %} - -{% if compilers.llvm %} -llvm/packages.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --explicit --format '{prefix}' {{ compilers.llvm.requires }})) -{% endif %} - -{% if compilers.get('llvm-amdgpu') %} -llvm-amdgpu/packages.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --explicit --format '{prefix}' {{ compilers.get('llvm-amdgpu').requires }})) -{% endif %} - -{% if compilers.get('intel-oneapi-compilers') %} -intel-oneapi-compilers/packages.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --explicit --format '{prefix}' {{ compilers.get('intel-oneapi-compilers').requires }})) -{% endif %} - -{% if compilers.nvhpc %} -nvhpc/packages.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --explicit --format '{prefix}' {{ compilers.nvhpc.requires }})) -{% endif %} - - - -include ../Make.inc - -# GNU Make isn't very smart about dependencies across included Makefiles, so we -# specify the order here by conditionally including them, when the dependent exists. -ifeq (,$(filter clean,$(MAKECMDGOALS))) - -include gcc/Makefile - -{% if compilers.llvm %} -ifneq (,$(wildcard gcc/Makefile)) -include llvm/Makefile -endif -{% endif %} - -{% if compilers.get('llvm-amdgpu') %} -ifneq (,$(wildcard gcc/Makefile)) -include llvm-amdgpu/Makefile -endif -{% endif %} - -{% if compilers.get('intel-oneapi-compilers') %} -ifneq (,$(wildcard gcc/Makefile)) -include intel-oneapi-compilers/Makefile -endif -{% endif %} - -{% if compilers.nvhpc %} -ifneq (,$(wildcard gcc/Makefile)) -include nvhpc/Makefile -endif -{% endif %} - - -endif diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments deleted file mode 100644 index 5a530232..00000000 --- a/stackinator/templates/Makefile.environments +++ /dev/null @@ -1,67 +0,0 @@ -{% set pipejoiner = joiner('|') %} --include ../Make.user - -MAKEFLAGS += --output-sync=recurse - -.PHONY: all .locks .packages.yaml - -all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} - - -# Ensure that spack.lock files are never removed as intermediate files -.locks:{% for env in environments %} {{ env }}/spack.lock{% endfor %} - -# Ensure that package yaml files are never removed as intermediate files... -.packages.yaml:{% for env in environments %} {{ env }}/packages.yaml{% endfor %} - -# Push built packages to a binary cache if a key has been provided -{% for env, config in environments.items() %} -{{ env }}/generated/build_cache: {{ env }}/generated/view_config -{% if push_to_cache %} - $(SPACK) --color=never -e ./{{ env }} buildcache create --rebuild-index --only=package alpscache \ - $$($(SPACK_HELPER) -e ./{{ env }} find --format '{name};{/hash};version={version}' \ - | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ - | grep -v -E 'version=git\.'\ - | cut -d ';' -f2)\ - | grep -Ev '^==> Fetching|^gpg:' -{% endif %} - touch $@ - -{% endfor %} - -# Create environment view where requested -{% for env, config in environments.items() %} -{{ env }}/generated/view_config: {{ env }}/generated/env -{% for view in config.views %} - $(SPACK) env activate --with-view {{ view.name }} --sh ./{{ env }} > $(STORE)/env/{{ view.name }}/activate.sh - $(BUILD_ROOT)/envvars.py view {% if view.extra.add_compilers %}--compilers=./{{ env }}/packages.yaml {% endif %} --prefix_paths="{{ view.extra.prefix_string }}" $(STORE)/env/{{ view.name }} $(BUILD_ROOT) -{% endfor %} - touch $@ - -{% endfor %} - - -{% for env in environments %}{{ env }}/config.yaml {% endfor %}: | store - $(SPACK) config --scope=user add config:install_tree:root:$(STORE) - -# Create the compilers.yaml configuration for each environment -{% for env, config in environments.items() %} -{{ env }}_PREFIX = {% for C in config.compiler %} $$($(SPACK_HELPER) -e ../compilers/{{ C }} find --explicit --format '{prefix}' {{ C.spec }}){% endfor %} - -{{ env }}/packages.yaml: - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $({{ env }}_PREFIX)) -{% if config.packages %} - $(SPACK) external find --not-buildable --scope=user {% for package in config.packages %} {{package}}{% endfor %} -{% endif %} - - -{% endfor %} - --include ../Make.inc - -ifeq (,$(filter clean,$(MAKECMDGOALS))) -{% for env in environments %} -include {{ env }}/Makefile -{% endfor %} -endif - diff --git a/stackinator/templates/Makefile.generate-config b/stackinator/templates/Makefile.generate-config index a53a1c8c..123f69ea 100644 --- a/stackinator/templates/Makefile.generate-config +++ b/stackinator/templates/Makefile.generate-config @@ -1,39 +1,30 @@ include ../Make.user CONFIG_DIR = $(STORE)/config -MODULE_DIR = $(BUILD_ROOT)/modules - -# These will be the prefixes of the GCCs, LLVMs and NVHPCs in the respective environments. -ALL_COMPILER_PREFIXES ={% for compiler in all_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --explicit --format='{prefix}' gcc llvm llvm-amdgpu nvhpc intel-oneapi){% endfor %} - - -COMPILER_PREFIXES ={% for compiler in release_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --explicit --format='{prefix}' gcc llvm llvm-amdgpu nvhpc intel-oneapi){% endfor %} +all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml{% if modules %} $(MODULE_DIR)/upstreams.yaml{% endif %} -all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml{% if modules %} $(MODULE_DIR)/upstreams.yaml $(MODULE_DIR)/compilers.yaml{% endif %} -# Generate the upstream configuration that will be provided by the mounted image $(CONFIG_DIR)/upstreams.yaml: - $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) - -# Copy the cluster-specific packages.yaml file to the configuration. -$(CONFIG_DIR)/packages.yaml: - # first create the directory, copy the base config and then update with compilers mkdir -p $(CONFIG_DIR) - install -m 644 $(BUILD_ROOT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(COMPILER_PREFIXES)) + $(SPACK) -e $(BUILD_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) + +# Merge the build packages.yaml with the installed compiler externals so that +# downstream Spack users can find and use the compilers without a compiler.yaml. +$(CONFIG_DIR)/packages.yaml: $(CONFIG_DIR)/upstreams.yaml + install -m 644 $(BUILD_ROOT)/packages.yaml $(CONFIG_DIR)/packages.yaml + $(SPACK) -e $(BUILD_ROOT) python $(BUILD_ROOT)/compiler-config.py \ + $(CONFIG_DIR)/packages.yaml \ + {{ compiler_names | join(' ') }} -# requires packages.yaml to ensure that the path $(CONFIG_DIR) has been created. $(CONFIG_DIR)/repos.yaml: $(CONFIG_DIR)/packages.yaml install -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml -# Generate a configuration used to generate the module files -# The configuration in CONFIG_DIR can't be used for this purpose, because a compilers.yaml -# that includes the bootstrap compiler is required to build the modules. -$(MODULE_DIR)/packages.yaml: # TODO probably this is going to be dropped - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(ALL_COMPILER_PREFIXES)) +{% if modules %} +MODULE_DIR = $(BUILD_ROOT)/modules $(MODULE_DIR)/upstreams.yaml: - $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) + $(SPACK) -e $(BUILD_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) +{% endif %} include ../Make.inc diff --git a/stackinator/templates/compilers.gcc.spack.yaml b/stackinator/templates/compilers.gcc.spack.yaml deleted file mode 100644 index 601d9512..00000000 --- a/stackinator/templates/compilers.gcc.spack.yaml +++ /dev/null @@ -1,25 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - packages: - gcc: - variants: [build_type=Release +bootstrap +profiled +strip ~binutils] - mpc: - variants: [libs=static] - gmp: - variants: [libs=static] - mpfr: - variants: [libs=static] - zstd: - variants: [libs=static] - zlib: - variants: [~shared] diff --git a/stackinator/templates/compilers.intel-oneapi-compilers.spack.yaml b/stackinator/templates/compilers.intel-oneapi-compilers.spack.yaml deleted file mode 100644 index e567b226..00000000 --- a/stackinator/templates/compilers.intel-oneapi-compilers.spack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - diff --git a/stackinator/templates/compilers.llvm-amdgpu.spack.yaml b/stackinator/templates/compilers.llvm-amdgpu.spack.yaml deleted file mode 100644 index e567b226..00000000 --- a/stackinator/templates/compilers.llvm-amdgpu.spack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - diff --git a/stackinator/templates/compilers.llvm.spack.yaml b/stackinator/templates/compilers.llvm.spack.yaml deleted file mode 100644 index e567b226..00000000 --- a/stackinator/templates/compilers.llvm.spack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - diff --git a/stackinator/templates/compilers.nvhpc.spack.yaml b/stackinator/templates/compilers.nvhpc.spack.yaml deleted file mode 100644 index e567b226..00000000 --- a/stackinator/templates/compilers.nvhpc.spack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - diff --git a/stackinator/templates/environments.spack.yaml b/stackinator/templates/environments.spack.yaml deleted file mode 100644 index 65a11855..00000000 --- a/stackinator/templates/environments.spack.yaml +++ /dev/null @@ -1,42 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - config: - deprecated: {{ config.deprecated }} - concretizer: - unify: {{ config.unify }} - reuse: false - duplicates: - strategy: {{ config.duplicates.strategy }} - specs: -{% for spec in config.specs %} - - '{{ spec }}' -{% endfor %} -{% if config.prefer or config.variants or config.mpi %} - packages: -{% if config.prefer or config.variants %} - all: - {% if config.prefer %} -{% set separator = joiner(', ') %} - prefer: [{% for c in config.prefer %}{{ separator() }}'{{ c }}'{% endfor %}] - {% endif %} - {% if config.variants %} -{% set separator = joiner(', ') %} - variants: [{% for v in config.variants %}{{ separator() }}'{{ v }}'{% endfor %}] - {% endif %} -{% endif %} - {% if config.mpi %} - mpi: - require: '{{ config.mpi }}' - {% endif %} -{% endif %} -{% if config.views %} - view: -{% for view in config.views %} - {{ view.name }}: - {{ view.config|py2yaml(6) }} -{% endfor %} -{% else %} - view: false -{% endif %} diff --git a/stackinator/templates/spack.yaml b/stackinator/templates/spack.yaml new file mode 100644 index 00000000..e9f217c5 --- /dev/null +++ b/stackinator/templates/spack.yaml @@ -0,0 +1,171 @@ +spack: + include: + - packages.yaml + - config.yaml + concretizer: + reuse: false + specs: + + # ---- Compiler groups ---- + # gcc is always built first using the system compiler. + - group: gcc + explicit: false + specs: +{% for spec in compilers.gcc.specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + packages: + gcc: + variants: [build_type=Release +bootstrap +profiled +strip ~binutils] + mpc: + variants: [libs=static] + gmp: + variants: [libs=static] + mpfr: + variants: [libs=static] + zstd: + variants: [libs=static] + zlib: + variants: [~shared] + +{% if compilers.nvhpc %} + - group: nvhpc + explicit: false + needs: [gcc] + specs: +{% for spec in compilers.nvhpc.specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + +{% endif %} +{% if compilers.llvm %} + - group: llvm + explicit: false + needs: [gcc] + specs: +{% for spec in compilers.llvm.specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + +{% endif %} +{% if compilers.get('llvm-amdgpu') %} + - group: llvm-amdgpu + explicit: false + needs: [gcc] + specs: +{% for spec in compilers['llvm-amdgpu'].specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + +{% endif %} +{% if compilers.get('intel-oneapi-compilers') %} + - group: intel-oneapi-compilers + explicit: false + needs: [gcc] + specs: +{% for spec in compilers['intel-oneapi-compilers'].specs %} + - '{{ spec }}' +{% endfor %} + override: + concretizer: + unify: when_possible + reuse: false + +{% endif %} + # ---- Internal tools group ---- + # squashfs is required to create the final squashfs image. + - group: uenv_tools + explicit: false + needs: [gcc] + override: + concretizer: + unify: true + reuse: false + specs: + - squashfs + + # ---- User environment groups ---- +{% for name, config in environments.items() %} + - group: {{ name }} + needs: [{{ config.compiler | join(', ') }}] + override: + config: + deprecated: {{ config.deprecated | string | lower }} + concretizer: + reuse: false + unify: {{ config.unify | string | lower }} + duplicates: + strategy: {{ config.duplicates.strategy }} +{% if config.prefer or config.variants or config.mpi %} + packages: +{% if config.prefer or config.variants %} + all: +{% if config.prefer %} + prefer: +{% for p in config.prefer %} + - '{{ p }}' +{% endfor %} +{% endif %} +{% if config.variants %} + variants: +{% for v in config.variants %} + - '{{ v }}' +{% endfor %} +{% endif %} +{% endif %} +{% if config.mpi %} + mpi: + require: '{{ config.mpi }}' +{% endif %} +{% endif %} + specs: +{% for spec in config.specs %} + - '{{ spec }}' +{% endfor %} + +{% endfor %} +{% if has_views %} + view: +{% for name, config in environments.items() %} +{% for view in config.views %} + {{ view.name }}: + root: '{{ view.config.root }}' + group: {{ name }} +{% if view.config.link != 'roots' %} + link: {{ view.config.link }} +{% endif %} +{% if view.config.get('exclude') %} + exclude: +{% for e in view.config.exclude %} + - '{{ e }}' +{% endfor %} +{% endif %} +{% if view.config.get('select') %} + select: +{% for s in view.config.select %} + - '{{ s }}' +{% endfor %} +{% endif %} +{% if view.config.get('projections') %} + projections: + {{ view.config.projections | py2yaml(8) }} +{% endif %} +{% endfor %} +{% endfor %} +{% endif %} diff --git a/unittests/recipes/base-nvgpu/config.yaml b/unittests/recipes/base-nvgpu/config.yaml index ccdd1f7c..161e231b 100644 --- a/unittests/recipes/base-nvgpu/config.yaml +++ b/unittests/recipes/base-nvgpu/config.yaml @@ -5,4 +5,4 @@ spack: commit: 6408b51 packages: repo: https://github.com/spack/spack-packages.git -version: 2 +version: 3 diff --git a/unittests/recipes/cache/config.yaml b/unittests/recipes/cache/config.yaml index a3c01cf7..b878749f 100644 --- a/unittests/recipes/cache/config.yaml +++ b/unittests/recipes/cache/config.yaml @@ -5,4 +5,4 @@ spack: commit: 6408b51 packages: repo: https://github.com/spack/spack-packages.git -version: 2 +version: 3 diff --git a/unittests/recipes/host-recipe/config.yaml b/unittests/recipes/host-recipe/config.yaml index 780cf031..79dfcd26 100644 --- a/unittests/recipes/host-recipe/config.yaml +++ b/unittests/recipes/host-recipe/config.yaml @@ -6,4 +6,4 @@ spack: repo: https://github.com/spack/spack.git packages: repo: https://github.com/spack/spack-packages.git -version: 2 +version: 3 diff --git a/unittests/recipes/with-repo/config.yaml b/unittests/recipes/with-repo/config.yaml index a3cb2940..1bd5a8a5 100644 --- a/unittests/recipes/with-repo/config.yaml +++ b/unittests/recipes/with-repo/config.yaml @@ -5,4 +5,4 @@ spack: commit: v21.0 packages: repo: https://github.com/spack/spack-packages.git -version: 2 +version: 3 diff --git a/unittests/test_schema.py b/unittests/test_schema.py index b30f73da..7c6b47a5 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -47,7 +47,7 @@ def test_config_yaml(yaml_path): # no spack:commit config = dedent(""" - version: 2 + version: 3 name: env-without-spack-commit spack: repo: https://github.com/spack/spack.git @@ -66,7 +66,7 @@ def test_config_yaml(yaml_path): # no spack:packages:commit config = dedent(""" - version: 2 + version: 3 name: env-without-spack-packages-commit spack: repo: https://github.com/spack/spack.git diff --git a/unittests/yaml/config.defaults.yaml b/unittests/yaml/config.defaults.yaml index 7197272a..3dcdca3b 100644 --- a/unittests/yaml/config.defaults.yaml +++ b/unittests/yaml/config.defaults.yaml @@ -10,4 +10,4 @@ spack: # default: None == no `git checkout` command #commit: 6408b51 #modules: True -version: 2 +version: 3 diff --git a/unittests/yaml/config.full.yaml b/unittests/yaml/config.full.yaml index f7891614..31c1f477 100644 --- a/unittests/yaml/config.full.yaml +++ b/unittests/yaml/config.full.yaml @@ -8,4 +8,4 @@ spack: commit: v2025.07.0 modules: False description: "a really useful environment" -version: 2 +version: 3 From a72631f170ec50b4589f81e12b2e72dfc20d294f Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 24 May 2026 10:33:33 +0200 Subject: [PATCH 02/15] more reformating --- stackinator/builder.py | 11 +++++++++-- stackinator/recipe.py | 4 +--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 011b7130..55d91c8d 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -211,7 +211,10 @@ def generate(self, recipe): "spack": self.path / "spack", } - for hook_name, hook_src in [("post-install", recipe.post_install_hook), ("pre-install", recipe.pre_install_hook)]: + for hook_name, hook_src in [ + ("post-install", recipe.post_install_hook), + ("pre-install", recipe.pre_install_hook), + ]: if hook_src is not None: self._logger.debug(f"installing {hook_name} script") jinja_recipe_env = jinja2.Environment(loader=jinja2.FileSystemLoader(recipe.path)) @@ -261,7 +264,11 @@ def generate(self, recipe): with (config_path / "repos.yaml").open("w") as f: repo_path = recipe.mount / "repos" / "spack_repo" / "alps" builtin_repo_path = recipe.mount / "repos" / "spack_repo" / "builtin" - f.write(repos_yaml_template.render(repo_path=repo_path.as_posix(), builtin_repo_path=builtin_repo_path.as_posix())) + f.write( + repos_yaml_template.render( + repo_path=repo_path.as_posix(), builtin_repo_path=builtin_repo_path.as_posix() + ) + ) f.write("\n") if recipe.mirror: diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 6601b414..4f7ef3ca 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -319,9 +319,7 @@ def generate_environment_specs(self, raw): mpi_name = match.group(1) supported_mpis = list(self.mpi_templates.keys()) if mpi_name not in supported_mpis: - raise Exception( - f"{mpi_name} is not a supported MPI: try one of {supported_mpis}." - ) + raise Exception(f"{mpi_name} is not a supported MPI: try one of {supported_mpis}.") else: raise Exception(f"{spec} is not a valid MPI spec") From 887e73990eac5ed4ff388c9b82fc1ee832fa8e29 Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 24 May 2026 15:09:53 +0200 Subject: [PATCH 03/15] concretize and build of simple env works - later steps fail --- stackinator/builder.py | 24 +++++++++---------- stackinator/templates/Make.user | 3 +++ stackinator/templates/Makefile | 18 +++++++------- .../templates/Makefile.generate-config | 9 ++++--- stackinator/templates/spack.yaml | 3 --- 5 files changed, 28 insertions(+), 29 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 55d91c8d..a43dd4d4 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -113,6 +113,8 @@ def generate(self, recipe): tmp_path = self.path / "tmp" self.path.mkdir(exist_ok=True, parents=True) + env_path = self.path / "env" + env_path.mkdir(exist_ok=True) store_path.mkdir(exist_ok=True) tmp_path.mkdir(exist_ok=True) @@ -152,19 +154,10 @@ def generate(self, recipe): ) # --- Write the unified spack.yaml --- - with (self.path / "spack.yaml").open("w") as f: + with (env_path / "spack.yaml").open("w") as f: f.write(recipe.spack_yaml) f.write("\n") - # --- Write packages.yaml (merged system + network + recipe packages) --- - with (self.path / "packages.yaml").open("w") as f: - f.write(yaml.dump(recipe.packages)) - - # --- Write config.yaml (install tree location) --- - config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}}} - with (self.path / "config.yaml").open("w") as f: - f.write(yaml.dump(config_yaml)) - # --- Write Makefile --- has_views = any(env_cfg["views"] for env_cfg in recipe.environments.values()) makefile_template = jinja_env.get_template("Makefile") @@ -255,11 +248,18 @@ def generate(self, recipe): with (repo_dst / "repo.yaml").open("w") as f: f.write("repo:\n namespace: alps\n api: v2.0\n") - # config/repos.yaml — consumed by SPACK_SYSTEM_CONFIG_PATH during the build - # and copied to store/config/repos.yaml by generate-config for downstream users + # config/ is the SPACK_SYSTEM_CONFIG_PATH scope: all files here are loaded + # automatically by every spack command, with or without -e. config_path = self.path / "config" config_path.mkdir(exist_ok=True) + with (config_path / "packages.yaml").open("w") as f: + f.write(yaml.dump(recipe.packages)) + + config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}}} + with (config_path / "config.yaml").open("w") as f: + f.write(yaml.dump(config_yaml)) + repos_yaml_template = jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: repo_path = recipe.mount / "repos" / "spack_repo" / "alps" diff --git a/stackinator/templates/Make.user b/stackinator/templates/Make.user index cb8acb03..b878d6cb 100644 --- a/stackinator/templates/Make.user +++ b/stackinator/templates/Make.user @@ -5,6 +5,9 @@ # This is the root of the software stack directory. BUILD_ROOT := {{ build_path }} +# Spack environment directory (spack.yaml, packages.yaml, config.yaml live here). +ENV_ROOT := $(BUILD_ROOT)/env + # What Spack should we use? SPACK := spack diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 75ca36ab..49497500 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -34,7 +34,7 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% if cache %} - $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) buildcache keys --install --trust + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache keys --install --trust {% if cache.key %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} @@ -42,17 +42,17 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} touch mirror-setup concretize: mirror-setup - $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) concretize + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) concretize touch concretize install: concretize - $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) install + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) install touch install {% if push_to_cache and cache.key %} cache-push: install - $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) buildcache create --rebuild-index --only=package alpscache \ - $$($(SANDBOX) $(SPACK_HELPER) -e $(BUILD_ROOT) find --format '{name};{/hash};version={version}' \ + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache create --rebuild-index --only=package alpscache \ + $$($(SANDBOX) $(SPACK_HELPER) -e $(ENV_ROOT) find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ | cut -d ';' -f2)\ @@ -62,7 +62,7 @@ cache-push: install # Generate compiler-config.yaml for use by view and generate-config steps. compiler-config.yaml: install - $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) python $(BUILD_ROOT)/compiler-config.py \ + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ $(BUILD_ROOT)/compiler-config.yaml \ {{ compiler_names | join(' ') }} @@ -71,7 +71,7 @@ compiler-config.yaml: install {% for view in config.views %} views/{{ view.name }}: install compiler-config.yaml mkdir -p $(dir $@) - $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) env activate --with-view {{ view.name }} --sh \ + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) env activate --with-view {{ view.name }} --sh \ > $(STORE)/env/{{ view.name }}/activate.sh $(SANDBOX) $(BUILD_ROOT)/envvars.py view \ {% if view.extra.add_compilers %}--compilers=$(BUILD_ROOT)/compiler-config.yaml {% endif %}\ @@ -91,7 +91,7 @@ generate-config: install compiler-config.yaml {% if modules %} modules-done: install generate-config - $(SANDBOX) $(SPACK) -e $(BUILD_ROOT) module tcl refresh --upstream-modules --delete-tree --yes-to-all + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) module tcl refresh --upstream-modules --delete-tree --yes-to-all touch modules-done {% endif %} @@ -115,7 +115,7 @@ store.squashfs: env-meta{% if post_install_hook %} post-install{% endif %}{% if $(SANDBOX) find $(STORE)/repos -type d -name __pycache__ -exec rm -r {} + $(SANDBOX) chmod -R a+rX $(STORE) $(SANDBOX) env -u SOURCE_DATE_EPOCH \ - "$$($(SANDBOX) $(SPACK_HELPER) -e $(BUILD_ROOT) find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" \ + "$$($(SANDBOX) $(SPACK_HELPER) -e $(ENV_ROOT) find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" \ $(STORE) $@ -force-uid nobody -force-gid nobody \ -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3 diff --git a/stackinator/templates/Makefile.generate-config b/stackinator/templates/Makefile.generate-config index 123f69ea..76e11d55 100644 --- a/stackinator/templates/Makefile.generate-config +++ b/stackinator/templates/Makefile.generate-config @@ -7,13 +7,12 @@ all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repo $(CONFIG_DIR)/upstreams.yaml: mkdir -p $(CONFIG_DIR) - $(SPACK) -e $(BUILD_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) + $(SPACK) -e $(ENV_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) -# Merge the build packages.yaml with the installed compiler externals so that +# Generate packages.yaml with the spack-built compiler externals so that # downstream Spack users can find and use the compilers without a compiler.yaml. $(CONFIG_DIR)/packages.yaml: $(CONFIG_DIR)/upstreams.yaml - install -m 644 $(BUILD_ROOT)/packages.yaml $(CONFIG_DIR)/packages.yaml - $(SPACK) -e $(BUILD_ROOT) python $(BUILD_ROOT)/compiler-config.py \ + $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ $(CONFIG_DIR)/packages.yaml \ {{ compiler_names | join(' ') }} @@ -24,7 +23,7 @@ $(CONFIG_DIR)/repos.yaml: $(CONFIG_DIR)/packages.yaml MODULE_DIR = $(BUILD_ROOT)/modules $(MODULE_DIR)/upstreams.yaml: - $(SPACK) -e $(BUILD_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) + $(SPACK) -e $(ENV_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) {% endif %} include ../Make.inc diff --git a/stackinator/templates/spack.yaml b/stackinator/templates/spack.yaml index e9f217c5..12ede57a 100644 --- a/stackinator/templates/spack.yaml +++ b/stackinator/templates/spack.yaml @@ -1,7 +1,4 @@ spack: - include: - - packages.yaml - - config.yaml concretizer: reuse: false specs: From b5f33488eeddb3f95a64315c8d9e7d57bb5a52ad Mon Sep 17 00:00:00 2001 From: bcumming Date: Sun, 24 May 2026 15:45:22 +0200 Subject: [PATCH 04/15] first e2e run --- stackinator/templates/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 49497500..1703b921 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -71,8 +71,8 @@ compiler-config.yaml: install {% for view in config.views %} views/{{ view.name }}: install compiler-config.yaml mkdir -p $(dir $@) - $(SANDBOX) $(SPACK) -e $(ENV_ROOT) env activate --with-view {{ view.name }} --sh \ - > $(STORE)/env/{{ view.name }}/activate.sh + $(SANDBOX) mkdir -p $(STORE)/env/{{ view.name }} + $(SANDBOX) sh -c '$(SPACK) env activate -d $(ENV_ROOT) --with-view {{ view.name }} --sh > $(STORE)/env/{{ view.name }}/activate.sh' $(SANDBOX) $(BUILD_ROOT)/envvars.py view \ {% if view.extra.add_compilers %}--compilers=$(BUILD_ROOT)/compiler-config.yaml {% endif %}\ --prefix_paths="{{ view.extra.prefix_string }}" \ @@ -87,7 +87,7 @@ views: {% for name, config in environments.items() %}{% for view in config.views touch views generate-config: install compiler-config.yaml - $(SANDBOX) $(MAKE) -C generate-config + $(SANDBOX) $(MAKE) -j1 -C generate-config {% if modules %} modules-done: install generate-config From 91922895b367313fa2bc249eba9e870dd3a423e6 Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 25 May 2026 09:34:19 +0200 Subject: [PATCH 05/15] enforce usage of user-requested gcc --- stackinator/recipe.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 4f7ef3ca..64391efa 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -336,13 +336,14 @@ def generate_environment_specs(self, raw): # Auto-generate a prefer constraint that pins the default compiler for name, config in environments.items(): if config["prefer"] is None: - compiler = config["compiler"][0] + compiler_key = config["compiler"][0] # spack uses a different name for the intel oneapi compilers # than the package that installs them. - if compiler == "intel-oneapi-compilers": - compiler = "oneapi" + compiler_name = "oneapi" if compiler_key == "intel-oneapi-compilers" else compiler_key + compiler_version = self.compilers[compiler_key]["version"] + versioned = f"{compiler_name}@{compiler_version}" config["prefer"] = [ - f"%[when=%c] c={compiler} %[when=%cxx] cxx={compiler} %[when=%fortran] fortran={compiler}" + f"%[when=%c] c={versioned} %[when=%cxx] cxx={versioned} %[when=%fortran] fortran={versioned}" ] # Build view metadata @@ -396,7 +397,7 @@ def generate_compiler_specs(self, raw): compilers = {} gcc_version = raw["gcc"]["version"] - compilers["gcc"] = {"specs": [f"gcc@{gcc_version} +bootstrap"]} + compilers["gcc"] = {"specs": [f"gcc@{gcc_version} +bootstrap"], "version": gcc_version} for name, spec_template in [ ("nvhpc", "nvhpc@{version} ~mpi~blas~lapack"), @@ -406,7 +407,7 @@ def generate_compiler_specs(self, raw): ]: if raw.get(name) is not None: version = raw[name]["version"] - compilers[name] = {"specs": [spec_template.format(version=version)]} + compilers[name] = {"specs": [spec_template.format(version=version)], "version": version} self.compilers = compilers From b3b9e189569fdaea82ef44bf5bb828ac181e008a Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 25 May 2026 10:12:07 +0200 Subject: [PATCH 06/15] add config:cleanup option to control garbage collection --- stackinator/builder.py | 1 + stackinator/schema/config.json | 5 +++++ stackinator/templates/Makefile | 12 ++++++++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index a43dd4d4..86ca6699 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -174,6 +174,7 @@ def generate(self, recipe): compiler_names=recipe.compiler_names, exclude_from_cache=["nvhpc", "cuda", "perl"], has_views=has_views, + cleanup=recipe.config["cleanup"], ) ) f.write("\n") diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 888d33c6..cbe274b6 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -68,6 +68,11 @@ "default": 1, "minimum": 1, "maximum": 3 + }, + "cleanup" : { + "type": "string", + "enum": ["none", "runtime", "build"], + "default": "none" } } } diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 1703b921..3f5b0a03 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -49,6 +49,14 @@ install: concretize $(SANDBOX) $(SPACK) -e $(ENV_ROOT) install touch install +cleanup: install +{% if cleanup == "build" %} + $(SANDBOX) $(SPACK) gc --yes-to-all --keep-build-dependencies --except-environment $(ENV_ROOT) +{% elif cleanup == "runtime" %} + $(SANDBOX) $(SPACK) gc --yes-to-all --except-environment $(ENV_ROOT) +{% endif %} + touch cleanup + {% if push_to_cache and cache.key %} cache-push: install $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache create --rebuild-index --only=package alpscache \ @@ -61,7 +69,7 @@ cache-push: install {% endif %} # Generate compiler-config.yaml for use by view and generate-config steps. -compiler-config.yaml: install +compiler-config.yaml: cleanup $(SANDBOX) $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ $(BUILD_ROOT)/compiler-config.yaml \ {{ compiler_names | join(' ') }} @@ -121,7 +129,7 @@ store.squashfs: env-meta{% if post_install_hook %} post-install{% endif %}{% if clean: rm -rf -- spack-version spack-setup{% if pre_install_hook %} pre-install{% endif %} mirror-setup \ - concretize install{% if push_to_cache and cache.key %} cache-push{% endif %} \ + concretize install cleanup{% if push_to_cache and cache.key %} cache-push{% endif %} \ compiler-config.yaml views generate-config/.done \ {% if modules %}modules-done {% endif %}env-meta{% if post_install_hook %} post-install{% endif %} \ store.squashfs spack-bootstrap-output From 487d8dd96a23cf1c9bac98620df2ccd554efcb30 Mon Sep 17 00:00:00 2001 From: bcumming Date: Mon, 25 May 2026 11:36:15 +0200 Subject: [PATCH 07/15] support system gcc --- stackinator/builder.py | 1 + stackinator/etc/compiler-config.py | 40 ++++++++++++++++++++++++++- stackinator/etc/envvars.py | 43 ++++++++++++++++++------------ stackinator/recipe.py | 28 ++++++++++++++----- stackinator/templates/Makefile | 8 +++--- stackinator/templates/spack.yaml | 24 ++++++++++++----- 6 files changed, 111 insertions(+), 33 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 86ca6699..4df0a219 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -175,6 +175,7 @@ def generate(self, recipe): exclude_from_cache=["nvhpc", "cuda", "perl"], has_views=has_views, cleanup=recipe.config["cleanup"], + system_gcc=recipe.system_gcc, ) ) f.write("\n") diff --git a/stackinator/etc/compiler-config.py b/stackinator/etc/compiler-config.py index f1d457bf..314edd64 100644 --- a/stackinator/etc/compiler-config.py +++ b/stackinator/etc/compiler-config.py @@ -79,10 +79,40 @@ def build_compiler_packages(compiler_names): return packages +def load_system_compiler_externals(system_packages_path): + """ + Read a packages.yaml and return entries that carry extra_attributes.compilers. + Used to surface system (external) compilers such as system gcc into the output. + """ + with open(system_packages_path) as fid: + data = yaml.safe_load(fid) or {} + + packages = {} + for pkg_name, pkg_data in data.get("packages", {}).items(): + if not isinstance(pkg_data, dict): + continue + compiler_externals = [ + e for e in pkg_data.get("externals", []) + if isinstance(e, dict) + and "extra_attributes" in e + and "compilers" in e["extra_attributes"] + ] + if compiler_externals: + packages[pkg_name] = {"externals": compiler_externals, "buildable": False} + print(f" compiler-config: found system compiler '{pkg_name}' in {system_packages_path}", file=sys.stderr) + + return packages + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("output", help="Path to packages.yaml to create or update") - parser.add_argument("compilers", nargs="+", help="Compiler package names to query") + parser.add_argument("compilers", nargs="*", help="Compiler package names to query") + parser.add_argument( + "--system-packages", + help="Path to a packages.yaml to read system compiler externals from", + default=None, + ) args = parser.parse_args() # Load existing content if the file already exists (merge mode). @@ -93,6 +123,14 @@ def main(): compiler_packages = build_compiler_packages(args.compilers) + # Pull in any system compiler externals (e.g. system gcc) that carry + # extra_attributes.compilers but are not in the spack store. + if args.system_packages and os.path.isfile(args.system_packages): + system_externals = load_system_compiler_externals(args.system_packages) + for pkg_name, pkg_data in system_externals.items(): + if pkg_name not in compiler_packages: + compiler_packages[pkg_name] = pkg_data + # Merge: compiler entries overwrite any existing entry for the same package name. merged = existing.copy() merged.setdefault("packages", {}).update(compiler_packages) diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index 218a77b6..55f29d9a 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -503,6 +503,11 @@ def view_impl(args): # remove all prefix path variable values that point to a location inside the build path. envvars.remove_root(args.build_path) + # Canonical symlink names for gcc role keys. For other compiler families + # (nvhpc, llvm, etc.) the binary names are already canonical, so we fall + # back to os.path.basename which gives the right name (nvc, clang, etc.). + _GCC_ROLE_CANONICAL = {"c": "gcc", "cxx": "g++", "fortran": "gfortran"} + if args.compilers is not None: if not os.path.isfile(args.compilers): print(f"error - compiler yaml file {args.compilers} does not exist") @@ -511,23 +516,27 @@ def view_impl(args): with open(args.compilers, "r") as file: data = yaml.safe_load(file) - compilers = [] - for p in data["packages"].values(): - for e in p["externals"]: - if "extra_attributes" in e: - c = e["extra_attributes"]["compilers"] - if c is not None: - compilers.append(c) - - for c in compilers: - source_paths = list(set([os.path.abspath(v) for _, v in c.items() if v is not None])) - target_paths = [os.path.join(bin_path, os.path.basename(f)) for f in source_paths] - for src, dst in zip(source_paths, target_paths): - print(f"creating compiler symlink: {src} -> {dst}") - if os.path.exists(dst): - print(f" first removing {dst}") - os.remove(dst) - os.symlink(src, dst) + for pkg_name, pkg_data in data["packages"].items(): + for e in pkg_data["externals"]: + if "extra_attributes" not in e: + continue + c = e["extra_attributes"].get("compilers") + if not c: + continue + for role, path in c.items(): + if path is None: + continue + src = os.path.abspath(path) + if pkg_name == "gcc": + link_name = _GCC_ROLE_CANONICAL.get(role, os.path.basename(src)) + else: + link_name = os.path.basename(src) + dst = os.path.join(bin_path, link_name) + print(f"creating compiler symlink: {src} -> {dst}") + if os.path.exists(dst): + print(f" first removing {dst}") + os.remove(dst) + os.symlink(src, dst) if args.prefix_paths: # get the root path of the env diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 64391efa..12e13951 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -289,8 +289,8 @@ def environment_view_meta(self): @property def compiler_names(self): - """Names of the compiler packages installed in this recipe.""" - return list(self.compilers.keys()) + """Names of the compiler packages installed in this recipe (excludes system gcc).""" + return [name for name, c in self.compilers.items() if not c.get("system", False)] # creates the self.environments field that describes the full specifications # for all of the environments sets, grouped in environments, from the raw @@ -333,19 +333,27 @@ def generate_environment_specs(self, raw): environments[name]["mpi"] = mpi_name environments[name]["specs"] += specs - # Auto-generate a prefer constraint that pins the default compiler + # Auto-generate a prefer constraint that pins the default compiler. + # For built compilers, include the version to distinguish from system externals. + # For system gcc, omit the version (there is only one gcc available). for name, config in environments.items(): if config["prefer"] is None: compiler_key = config["compiler"][0] # spack uses a different name for the intel oneapi compilers # than the package that installs them. compiler_name = "oneapi" if compiler_key == "intel-oneapi-compilers" else compiler_key - compiler_version = self.compilers[compiler_key]["version"] - versioned = f"{compiler_name}@{compiler_version}" + compiler_version = self.compilers[compiler_key].get("version") + versioned = f"{compiler_name}@{compiler_version}" if compiler_version else compiler_name config["prefer"] = [ f"%[when=%c] c={versioned} %[when=%cxx] cxx={versioned} %[when=%fortran] fortran={versioned}" ] + # Compute spec group needs: only compilers with actual (non-system) spec groups. + for name, config in environments.items(): + config["needs"] = [ + c for c in config["compiler"] if not self.compilers.get(c, {}).get("system", False) + ] + # Build view metadata env_names = set() for name, config in environments.items(): @@ -397,7 +405,10 @@ def generate_compiler_specs(self, raw): compilers = {} gcc_version = raw["gcc"]["version"] - compilers["gcc"] = {"specs": [f"gcc@{gcc_version} +bootstrap"], "version": gcc_version} + if gcc_version == "system": + compilers["gcc"] = {"system": True} + else: + compilers["gcc"] = {"specs": [f"gcc@{gcc_version} +bootstrap"], "version": gcc_version} for name, spec_template in [ ("nvhpc", "nvhpc@{version} ~mpi~blas~lapack"), @@ -411,6 +422,10 @@ def generate_compiler_specs(self, raw): self.compilers = compilers + @property + def system_gcc(self): + return self.compilers.get("gcc", {}).get("system", False) + # The path of the default configuration for the target system/cluster @property def system_config_path(self): @@ -449,4 +464,5 @@ def spack_yaml(self): environments=self.environments, store=self.mount, has_views=has_views, + system_gcc=self.system_gcc, ) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 3f5b0a03..4a1098bb 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -41,7 +41,7 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} touch mirror-setup -concretize: mirror-setup +concretize: mirror-setup env/spack.yaml $(SANDBOX) $(SPACK) -e $(ENV_ROOT) concretize touch concretize @@ -71,8 +71,10 @@ cache-push: install # Generate compiler-config.yaml for use by view and generate-config steps. compiler-config.yaml: cleanup $(SANDBOX) $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ - $(BUILD_ROOT)/compiler-config.yaml \ - {{ compiler_names | join(' ') }} +{% if system_gcc %} + --system-packages=$(BUILD_ROOT)/config/packages.yaml \ +{% endif %} + $(BUILD_ROOT)/compiler-config.yaml{% if compiler_names %} {{ compiler_names | join(' ') }}{% endif %} # Generate activate.sh and env.json for each environment view. {% for name, config in environments.items() %} diff --git a/stackinator/templates/spack.yaml b/stackinator/templates/spack.yaml index 12ede57a..7c402aac 100644 --- a/stackinator/templates/spack.yaml +++ b/stackinator/templates/spack.yaml @@ -4,7 +4,8 @@ spack: specs: # ---- Compiler groups ---- - # gcc is always built first using the system compiler. +{% if not system_gcc %} + # gcc is built first using the system compiler. - group: gcc explicit: false specs: @@ -29,10 +30,13 @@ spack: zlib: variants: [~shared] +{% endif %} {% if compilers.nvhpc %} - group: nvhpc explicit: false +{% if not system_gcc %} needs: [gcc] +{% endif %} specs: {% for spec in compilers.nvhpc.specs %} - '{{ spec }}' @@ -46,7 +50,9 @@ spack: {% if compilers.llvm %} - group: llvm explicit: false +{% if not system_gcc %} needs: [gcc] +{% endif %} specs: {% for spec in compilers.llvm.specs %} - '{{ spec }}' @@ -60,7 +66,9 @@ spack: {% if compilers.get('llvm-amdgpu') %} - group: llvm-amdgpu explicit: false +{% if not system_gcc %} needs: [gcc] +{% endif %} specs: {% for spec in compilers['llvm-amdgpu'].specs %} - '{{ spec }}' @@ -74,7 +82,9 @@ spack: {% if compilers.get('intel-oneapi-compilers') %} - group: intel-oneapi-compilers explicit: false +{% if not system_gcc %} needs: [gcc] +{% endif %} specs: {% for spec in compilers['intel-oneapi-compilers'].specs %} - '{{ spec }}' @@ -89,7 +99,9 @@ spack: # squashfs is required to create the final squashfs image. - group: uenv_tools explicit: false +{% if not system_gcc %} needs: [gcc] +{% endif %} override: concretizer: unify: true @@ -100,7 +112,9 @@ spack: # ---- User environment groups ---- {% for name, config in environments.items() %} - group: {{ name }} - needs: [{{ config.compiler | join(', ') }}] +{% if config.needs %} + needs: [{{ config.needs | join(', ') }}] +{% endif %} override: config: deprecated: {{ config.deprecated | string | lower }} @@ -143,10 +157,8 @@ spack: {% for view in config.views %} {{ view.name }}: root: '{{ view.config.root }}' - group: {{ name }} -{% if view.config.link != 'roots' %} - link: {{ view.config.link }} -{% endif %} + group: '{{ name }}' + link: '{{ view.config.link }}' {% if view.config.get('exclude') %} exclude: {% for e in view.config.exclude %} From 93857359e9f8ef3819c39bf6c40518fe17fa65a9 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 11 Jun 2026 12:54:17 +0200 Subject: [PATCH 08/15] accept gpg keys without user interaction --- stackinator/templates/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 4a1098bb..ae1782f0 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -34,9 +34,9 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% if cache %} - $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache keys --install --trust + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache keys --install --trust --yes-to-all {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} + $(SANDBOX) $(SPACK) gpg trust --yes-to-all {{ cache.key }} {% endif %} {% endif %} touch mirror-setup From 77e3d3cba4548b5ae292bef07ba45b7f90bcabc1 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 11 Jun 2026 12:55:06 +0200 Subject: [PATCH 09/15] use the old installer (for now) to work with Python 3.6 --- stackinator/builder.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 4df0a219..5545c956 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -258,7 +258,11 @@ def generate(self, recipe): with (config_path / "packages.yaml").open("w") as f: f.write(yaml.dump(recipe.packages)) - config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}}} + # Force the legacy ("old") installer: the new spack 1.2 installer drives a + # live TUI via selectors/pipes/non-blocking fds that fails with EBADF under + # the non-interactive `make` build on older Cray/SLES stacks (system Python + # 3.6). The TUI is pointless for a batch build in any case. + config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}, "installer": "old"}} with (config_path / "config.yaml").open("w") as f: f.write(yaml.dump(config_yaml)) @@ -333,9 +337,10 @@ def generate(self, recipe): meta_extra_path = meta_path / "extra" if meta_extra_path.exists(): shutil.rmtree(meta_extra_path) - meta_extra_path.mkdir(exist_ok=True) if recipe.user_extra is not None: install(recipe.user_extra, meta_extra_path) + else: + meta_extra_path.mkdir() # --- debug helper --- debug_template = jinja_env.get_template("stack-debug.sh") From 48e4e3d4c41efa50539311db7e6a31199b7a6eb4 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 11 Jun 2026 17:20:27 +0200 Subject: [PATCH 10/15] fix compatibility with new installer --- docs/building.md | 15 ++++++++++++++- stackinator/builder.py | 6 +----- stackinator/main.py | 3 ++- stackinator/templates/Make.user | 7 +++++++ stackinator/templates/Makefile | 33 ++++++++++++++++++++------------ stackinator/templates/spack.yaml | 12 +----------- 6 files changed, 46 insertions(+), 30 deletions(-) diff --git a/docs/building.md b/docs/building.md index c2d191ee..a518aa5e 100644 --- a/docs/building.md +++ b/docs/building.md @@ -12,11 +12,24 @@ stack-config --build $BUILD_PATH ... # perform the build cd $BUILD_PATH -env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make modules store.squashfs -j32 +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make modules store.squashfs NJOBS=32 ``` The call to `make` is wrapped with with `env --ignore-env` to unset all environment variables, to improve reproducability of builds. +## Controlling build parallelism + +The number of packages and compilation jobs that Spack runs concurrently is set with the `NJOBS` make variable: + +``` +make store.squashfs NJOBS=64 +``` + +Set `NJOBS` to the number of cores available on the build node (it defaults to `16` if not provided). + +!!! note + Do not use `make -jN` to control build parallelism. The `install` step clears `MAKEFLAGS` before invoking Spack — this avoids a crash with GNU make older than 4.4, whose legacy file-descriptor jobserver Spack mishandles — so the outer `make -j` flag does not reach Spack. `NJOBS` is passed to Spack as `spack install --jobs`, and is the only flag that governs build parallelism. + Build times for stacks typically vary between 30 minutes to 3 hours, depending on the specific packages that have to be built. Using [build caches](build-caches.md) and building in shared memory (see below) are the most effective methods to speed up builds. diff --git a/stackinator/builder.py b/stackinator/builder.py index 5545c956..317d53eb 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -258,11 +258,7 @@ def generate(self, recipe): with (config_path / "packages.yaml").open("w") as f: f.write(yaml.dump(recipe.packages)) - # Force the legacy ("old") installer: the new spack 1.2 installer drives a - # live TUI via selectors/pipes/non-blocking fds that fails with EBADF under - # the non-interactive `make` build on older Cray/SLES stacks (system Python - # 3.6). The TUI is pointless for a batch build in any case. - config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}, "installer": "old"}} + config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}}} with (config_path / "config.yaml").open("w") as f: f.write(yaml.dump(config_yaml)) diff --git a/stackinator/main.py b/stackinator/main.py index 44406215..51aeec3a 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -111,7 +111,8 @@ def main(): root_logger.info("\nConfiguration finished, run the following to build the environment:\n") root_logger.info(f"cd {builder.path}") root_logger.info( - "env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin HOME=$HOME make store.squashfs -j32" + "env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin HOME=$HOME " + "make store.squashfs NJOBS=32" ) root_logger.info(f"see logfile for more information {logfile}") return 0 diff --git a/stackinator/templates/Make.user b/stackinator/templates/Make.user index b878d6cb..55665638 100644 --- a/stackinator/templates/Make.user +++ b/stackinator/templates/Make.user @@ -54,6 +54,13 @@ export SPACK_USER_CACHE_PATH := $(BUILD_ROOT)/cache export SPACK_INSTALL_FLAGS := --verbose {% endif %} +# Number of concurrent build jobs for `spack install`. The new installer treats this as a +# global cap across all packages building at once. +# Set it explicitly because the install recipe clears MAKEFLAGS (see the Makefile), +# so the outer `make -j` no longer reaches spack. +# Override on the build node, e.g. `make store.squashfs NJOBS=64`. +NJOBS ?= 32 + # Reproducibility export LC_ALL := en_US.UTF-8 export TZ := UTC diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index ae1782f0..84dad23e 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -45,28 +45,37 @@ concretize: mirror-setup env/spack.yaml $(SANDBOX) $(SPACK) -e $(ENV_ROOT) concretize touch concretize +# Clear MAKEFLAGS for the install. When run with `make -j`, GNU make advertises a +# jobserver in MAKEFLAGS. GNU make < 4.4 uses the legacy fd form (`--jobserver-auth=R,W`) +# and closes those fds before running a non-recursive recipe, so they are invalid in the +# spack process. Spack's jobserver probe validates them with `fcntl(fd, F_GETFD) != -1`, +# a C idiom that in Python *raises* OSError(EBADF) instead of returning -1 - so the +# install aborts with `[Errno 9] Bad file descriptor` (independent of Python version). +# Hiding MAKEFLAGS makes spack create its own FIFO jobserver sized by `config:build_jobs`. +# Outer-make `-j` no longer governs spack; set build_jobs to control parallelism (or use +# GNU make >= 4.4, whose `fifo:` jobserver spack parses correctly). install: concretize - $(SANDBOX) $(SPACK) -e $(ENV_ROOT) install + MAKEFLAGS= $(SANDBOX) $(SPACK) -e $(ENV_ROOT) install --jobs $(NJOBS) touch install -cleanup: install -{% if cleanup == "build" %} - $(SANDBOX) $(SPACK) gc --yes-to-all --keep-build-dependencies --except-environment $(ENV_ROOT) -{% elif cleanup == "runtime" %} - $(SANDBOX) $(SPACK) gc --yes-to-all --except-environment $(ENV_ROOT) -{% endif %} - touch cleanup - -{% if push_to_cache and cache.key %} cache-push: install +{% if push_to_cache and cache.key %} $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache create --rebuild-index --only=package alpscache \ $$($(SANDBOX) $(SPACK_HELPER) -e $(ENV_ROOT) find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ - | cut -d ';' -f2)\ - | grep -Ev '^==> Fetching|^gpg:' + | cut -d ';' -f2) 2>&1 \ + | awk -v f="$(BUILD_ROOT)/gpg.log" '/^gpg:/{print > f; next} /^==> Fetching/{next} {print; fflush()}' +{% endif %} touch cache-push + +cleanup: cache-push +{% if cleanup == "build" %} + $(SANDBOX) $(SPACK) gc --yes-to-all --keep-build-dependencies --except-environment $(ENV_ROOT) +{% elif cleanup == "runtime" %} + $(SANDBOX) $(SPACK) gc --yes-to-all --except-environment $(ENV_ROOT) {% endif %} + touch cleanup # Generate compiler-config.yaml for use by view and generate-config steps. compiler-config.yaml: cleanup diff --git a/stackinator/templates/spack.yaml b/stackinator/templates/spack.yaml index 7c402aac..9955000b 100644 --- a/stackinator/templates/spack.yaml +++ b/stackinator/templates/spack.yaml @@ -18,17 +18,7 @@ spack: reuse: false packages: gcc: - variants: [build_type=Release +bootstrap +profiled +strip ~binutils] - mpc: - variants: [libs=static] - gmp: - variants: [libs=static] - mpfr: - variants: [libs=static] - zstd: - variants: [libs=static] - zlib: - variants: [~shared] + variants: [build_type=Release +bootstrap +strip ~binutils] {% endif %} {% if compilers.nvhpc %} From 71cdcbd5cf0f1c264fa23183f30deaaf3444dfb5 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 18 Jun 2026 14:30:32 +0200 Subject: [PATCH 11/15] wip --- stackinator/builder.py | 176 +++++------------------------------------ 1 file changed, 21 insertions(+), 155 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 77e99d11..4d8c963d 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -136,29 +136,6 @@ def generate(self, recipe): spack_git_commit = self._git_clone( "spack", spack["repo"], spack["commit"], spack_path) - -<< << << < HEAD - # Clone spack-packages - spack_packages = spack["packages"] - spack_packages_path = self.path / "spack-packages" - spack_packages_git_commit = self._git_clone( - "spack-packages", - spack_packages["repo"], - spack_packages["commit"], - spack_packages_path, - ) - - spack_meta = { - "url": spack["repo"], - "ref": spack["commit"], - "commit": spack_git_commit, - "packages_url": spack_packages["repo"], - "packages_ref": spack_packages["commit"], - "packages_commit": spack_packages_git_commit, -== == == = - spack_git_commit_result = self._git_clone( - "spack", spack_repo, spack_commit, spack_path) - package_repos = recipe.spack_package_repos for pkg_repo in package_repos: pkg_repo["path"] = self.path / "repos" / pkg_repo["name"] @@ -166,11 +143,10 @@ def generate(self, recipe): pkg_repo["name"], pkg_repo["url"], pkg_repo["ref"], pkg_repo["path"]) spack_meta = { - "url": spack_repo, - "ref": spack_commit, - "commit": spack_git_commit_result, + "url": spack["repo"], + "ref": spack["commit"], + "commit": spack_git_commit, "packages": package_repos, ->>>>>> > main } # Jinja environment for templates @@ -181,22 +157,11 @@ def generate(self, recipe): lstrip_blocks=True, ) -<< << << < HEAD # --- Write the unified spack.yaml --- with (env_path / "spack.yaml").open("w") as f: f.write(recipe.spack_yaml) f.write("\n") - # --- Write Makefile --- - has_views = any(env_cfg["views"] - for env_cfg in recipe.environments.values()) - makefile_template = jinja_env.get_template("Makefile") - with (self.path / "Makefile").open("w") as f: - f.write( - makefile_template.render( - cache=recipe.mirror, - push_to_cache=recipe.mirror is not None, -== == === # Write the spack mirror config artifacts (mirrors.yaml, bootstrap config, # and the relocated gpg keys) into the config scope. These were fully # resolved and validated by the recipe, so we just write the bytes. This @@ -207,7 +172,9 @@ def generate(self, recipe): dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(content) - # generate top level makefiles + # --- Write Makefile --- + has_views = any(env_cfg["views"] + for env_cfg in recipe.environments.values()) makefile_template=jinja_env.get_template("Makefile") # Extract module types that were configured in recipe.py @@ -220,20 +187,16 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( ->>>>>> > main modules=recipe.with_modules, module_types=module_types, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_meta=spack_meta, -<< << << < HEAD environments=recipe.environments, compiler_names=recipe.compiler_names, -== == === gpg_keys=recipe.mirrors.gpg_key_paths(config_path), cache=recipe.build_cache_mirror, buildcache_push=recipe.push_to_build_cache, ->>>>>> > main exclude_from_cache=["nvhpc", "cuda", "perl"], has_views=has_views, cleanup=recipe.config["cleanup"], @@ -283,82 +246,29 @@ def generate(self, recipe): f.write("\n") os.chmod(hook_dst, os.stat(hook_dst).st_mode | stat.S_IEXEC) -<< << << < HEAD - # --- Build the consolidated 'alps' spack package repo --- - # Precedence (highest first): recipe/repo > cluster repos.yaml entries > spack builtin - repos=[] - if recipe.spack_repo is not None: - self._logger.debug(f"adding recipe spack package repo: { - recipe.spack_repo}") - repos.append(recipe.spack_repo) - - repo_yaml_path=recipe.system_config_path / "repos.yaml" - if repo_yaml_path.exists(): - with repo_yaml_path.open() as fid: -== == === - with post_hook_destination.open("w") as f: - f.write(post_hook_template.render(env=hook_env, verbose=False)) - f.write("\n") - - os.chmod( - post_hook_destination, - os.stat(post_hook_destination).st_mode | stat.S_IEXEC, - ) - - # copy pre install hook file, if provided - pre_hook=recipe.pre_install_hook - if pre_hook is not None: - self._logger.debug("installing pre-install-hook script") - jinja_recipe_env=jinja2.Environment( - loader=jinja2.FileSystemLoader(recipe.path)) - pre_hook_template=jinja_recipe_env.get_template("pre-install") - pre_hook_destination=store_path / "pre-install-hook" - - with pre_hook_destination.open("w") as f: - f.write(pre_hook_template.render(env=hook_env, verbose=False)) - f.write("\n") - - os.chmod( - pre_hook_destination, - os.stat(pre_hook_destination).st_mode | stat.S_IEXEC, - ) - - # Generate the system configuration: the compilers, environments, etc. - # that are defined for the target cluster. - packages_path=config_path / "packages.yaml" - # the packages.yaml configuration that will be used when building all environments # - the system packages.yaml with gcc removed # - plus additional packages provided by the recipe - global_packages_yaml=yaml.dump(recipe.packages["global"]) - global_packages_path=config_path / "packages.yaml" - with global_packages_path.open("w") as fid: - fid.write(global_packages_yaml) + with (config_path / "packages.yaml").open("w") as f: + f.write(yaml.dump(recipe.packages)) + + config_yaml={"config": {"install_tree": {"root": str(recipe.mount)}}} + with (config_path / "config.yaml").open("w") as f: + f.write(yaml.dump(config_yaml)) # Add custom spack package recipes, configured via Spack repos. - # Step 1: copy Spack repos to store_path where they will be used to - # build the stack, and then be part of the upstream provided - # to users of the stack. - # - # Packages in the recipe are prioritised over cluster specific packages, - # etc. The order of preference from highest to lowest is: - # - # 3. recipe/repo - # 2. cluster-config/repos.yaml - # - if the repos.yaml file exists it will contain a list of relative paths - # to search for package - # 1. package repos from config.yaml in the order specified (typically - # only spack-packages builtin repo) - - # Build a list of repos with packages to install from system config and recipe. + # Build a list of repos with packages to install from system config. + # Packages in the recipe are prioritised over cluster specific packages. + # Order of preference from highest to lowest: + # 3. recipe/repo + # 2. cluster-config/repos.yaml entries + # 1. package repos from config.yaml (e.g. spack-packages builtin) repos=[] # look for repos.yaml file in the system configuration - repo_yaml=recipe.system_config_path / "repos.yaml" - if repo_yaml.exists() and repo_yaml.is_file(): - # open repos.yaml file and reat the list of repos - with repo_yaml.open() as fid: ->>>>>> > main + repo_yaml_path=recipe.system_config_path / "repos.yaml" + if repo_yaml_path.exists() and repo_yaml_path.is_file(): + with repo_yaml_path.open() as fid: raw=yaml.load(fid, Loader=yaml.Loader) for rel_path in raw["repos"]: repo_path=(recipe.system_config_path / rel_path).resolve() @@ -372,13 +282,10 @@ def generate(self, recipe): raise RuntimeError( "invalid system-provided package repository") -<< << << < HEAD -== == === self._logger.debug(f"full list of system spack package repos: {repos}") # Delete the store/repo path, if it already exists. # Do this so that incremental builds (though not officially supported) won't break if a repo is updated. ->> >>>> > main repos_path=store_path / "repos" / "spack_repo" repo_dst=repos_path / "alps" if repo_dst.exists(): @@ -386,22 +293,6 @@ def generate(self, recipe): pkg_dst=repo_dst / "packages" pkg_dst.mkdir(mode=0o755, parents=True) -<< << << < HEAD - with (repo_dst / "repo.yaml").open("w") as f: - f.write("repo:\n namespace: alps\n api: v2.0\n") - - # config/ is the SPACK_SYSTEM_CONFIG_PATH scope: all files here are loaded - # automatically by every spack command, with or without -e. - config_path=self.path / "config" - config_path.mkdir(exist_ok=True) - - with (config_path / "packages.yaml").open("w") as f: - f.write(yaml.dump(recipe.packages)) - - config_yaml={"config": {"install_tree": {"root": str(recipe.mount)}}} - with (config_path / "config.yaml").open("w") as f: - f.write(yaml.dump(config_yaml)) -== == === # create the repository step 2: create the repo.yaml file that # configures the alps repo with (repo_dst / "repo.yaml").open("w") as f: @@ -431,7 +322,6 @@ def generate(self, recipe): self._logger.debug(f" installing recipe package { pkg_path} to {recipe_pkg_dst}") install(pkg_path, dst) ->> >>>> > main repos_yaml_template=jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: @@ -446,31 +336,15 @@ def generate(self, recipe): ] f.write( repos_yaml_template.render( -<< << << < HEAD - repo_path=repo_path.as_posix(), builtin_repo_path=builtin_repo_path.as_posix() -== == === repo_path=repo_path.as_posix(), package_repos=package_repos, recipe_repo_path=recipe_repo_path.as_posix(), has_recipe_repo=has_recipe_repo, verbose=False, ->>>>>> > main ) ) f.write("\n") -<< << << < HEAD - if recipe.mirror: - with (config_path / "mirrors.yaml").open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) - - # Copy package definitions into the alps repo (recipe > site > nothing) - for repo_src in repos: - for pkg_path in (repo_src / "packages").iterdir(): - dst=pkg_dst / pkg_path.name - if pkg_path.is_dir() and not dst.exists(): - install(pkg_path, dst) -== == === # Iterate over the alps and recipe repositories copying their contents # to the final repo locations. Because of the order of repos in the # repos.yaml config file, recipe packages have precedence. @@ -499,14 +373,6 @@ def generate(self, recipe): self._logger.debug(f"{dst_path} exists ... deleting") shutil.rmtree(dst_path) install(src_path, dst_path) ->> >>>> > main - - # Copy builtin repo from spack-packages - spack_builtin_src=spack_packages_path / "repos" / "spack_repo" / "builtin" - spack_builtin_dst=store_path / "repos" / "spack_repo" / "builtin" - if spack_builtin_dst.exists(): - shutil.rmtree(spack_builtin_dst) - install(spack_builtin_src, spack_builtin_dst) # --- generate-config subdirectory --- generate_config_path=self.path / "generate-config" From f8a33f89d6b8d6fda9d420ceb14a82cb7658d227 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 18 Jun 2026 16:24:34 +0200 Subject: [PATCH 12/15] fix small bugs; make output of make easier to read --- stackinator/builder.py | 135 ++++++++++++++------------------- stackinator/etc/Make.inc | 11 +++ stackinator/etc/envvars.py | 2 - stackinator/templates/Makefile | 18 ++++- 4 files changed, 87 insertions(+), 79 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 4d8c963d..a061e0aa 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -115,8 +115,7 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): - store_path = self.path / \ - "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) + store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" config_path = self.path / "config" @@ -133,14 +132,12 @@ def generate(self, recipe): # Clone spack spack = recipe.config["spack"] spack_path = self.path / "spack" - spack_git_commit = self._git_clone( - "spack", spack["repo"], spack["commit"], spack_path) + spack_git_commit = self._git_clone("spack", spack["repo"], spack["commit"], spack_path) package_repos = recipe.spack_package_repos for pkg_repo in package_repos: pkg_repo["path"] = self.path / "repos" / pkg_repo["name"] - pkg_repo["commit"] = self._git_clone( - pkg_repo["name"], pkg_repo["url"], pkg_repo["ref"], pkg_repo["path"]) + pkg_repo["commit"] = self._git_clone(pkg_repo["name"], pkg_repo["url"], pkg_repo["ref"], pkg_repo["path"]) spack_meta = { "url": spack["repo"], @@ -166,23 +163,20 @@ def generate(self, recipe): # and the relocated gpg keys) into the config scope. These were fully # resolved and validated by the recipe, so we just write the bytes. This # must precede the Makefile render, which references the gpg key paths. - self._logger.debug( - f"Writing the spack mirror configs to '{config_path}'") + self._logger.debug(f"Writing the spack mirror configs to '{config_path}'") for dest, content in recipe.mirrors.config_files(config_path).items(): dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(content) # --- Write Makefile --- - has_views = any(env_cfg["views"] - for env_cfg in recipe.environments.values()) - makefile_template=jinja_env.get_template("Makefile") + has_views = any(env_cfg["views"] for env_cfg in recipe.environments.values()) + makefile_template = jinja_env.get_template("Makefile") # Extract module types that were configured in recipe.py - module_types=[] + module_types = [] if recipe.with_modules and recipe.modules: - roots=recipe.modules.get("modules", {}).get( - "default", {}).get("roots", {}) - module_types=list(roots.keys()) + roots = recipe.modules.get("modules", {}).get("default", {}).get("roots", {}) + module_types = list(roots.keys()) with (self.path / "Makefile").open("w") as f: f.write( @@ -206,7 +200,7 @@ def generate(self, recipe): f.write("\n") # --- Write Make.user --- - make_user_template=jinja_env.get_template("Make.user") + make_user_template = jinja_env.get_template("Make.user") with (self.path / "Make.user").open("w") as f: f.write( make_user_template.render( @@ -219,12 +213,12 @@ def generate(self, recipe): f.write("\n") # --- Copy static files from etc/ --- - etc_path=self.root / "etc" + etc_path = self.root / "etc" for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py", "compiler-config.py"]: shutil.copy2(etc_path / f_etc, self.path / f_etc) # --- Install hooks if provided --- - hook_env={ + hook_env = { "mount": recipe.mount, "config": recipe.mount / "config", "build": self.path, @@ -237,10 +231,9 @@ def generate(self, recipe): ]: if hook_src is not None: self._logger.debug(f"installing {hook_name} script") - jinja_recipe_env=jinja2.Environment( - loader=jinja2.FileSystemLoader(recipe.path)) - hook_template=jinja_recipe_env.get_template(hook_src.name) - hook_dst=store_path / f"{hook_name}-hook" + jinja_recipe_env = jinja2.Environment(loader=jinja2.FileSystemLoader(recipe.path)) + hook_template = jinja_recipe_env.get_template(hook_src.name) + hook_dst = store_path / f"{hook_name}-hook" with hook_dst.open("w") as f: f.write(hook_template.render(env=hook_env, verbose=False)) f.write("\n") @@ -252,7 +245,7 @@ def generate(self, recipe): with (config_path / "packages.yaml").open("w") as f: f.write(yaml.dump(recipe.packages)) - config_yaml={"config": {"install_tree": {"root": str(recipe.mount)}}} + config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}}} with (config_path / "config.yaml").open("w") as f: f.write(yaml.dump(config_yaml)) @@ -263,34 +256,31 @@ def generate(self, recipe): # 3. recipe/repo # 2. cluster-config/repos.yaml entries # 1. package repos from config.yaml (e.g. spack-packages builtin) - repos=[] + repos = [] # look for repos.yaml file in the system configuration - repo_yaml_path=recipe.system_config_path / "repos.yaml" + repo_yaml_path = recipe.system_config_path / "repos.yaml" if repo_yaml_path.exists() and repo_yaml_path.is_file(): with repo_yaml_path.open() as fid: - raw=yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid, Loader=yaml.Loader) for rel_path in raw["repos"]: - repo_path=(recipe.system_config_path / rel_path).resolve() + repo_path = (recipe.system_config_path / rel_path).resolve() if spack_util.is_repo(repo_path): repos.append(repo_path) - self._logger.debug( - f"adding site spack package repo: {repo_path}") + self._logger.debug(f"adding site spack package repo: {repo_path}") else: - self._logger.error(f"{repo_path} from { - repo_yaml_path} is not a spack package repository") - raise RuntimeError( - "invalid system-provided package repository") + self._logger.error(f"{repo_path} from {repo_yaml_path} is not a spack package repository") + raise RuntimeError("invalid system-provided package repository") self._logger.debug(f"full list of system spack package repos: {repos}") # Delete the store/repo path, if it already exists. # Do this so that incremental builds (though not officially supported) won't break if a repo is updated. - repos_path=store_path / "repos" / "spack_repo" - repo_dst=repos_path / "alps" + repos_path = store_path / "repos" / "spack_repo" + repo_dst = repos_path / "alps" if repo_dst.exists(): shutil.rmtree(repo_dst) - pkg_dst=repo_dst / "packages" + pkg_dst = repo_dst / "packages" pkg_dst.mkdir(mode=0o755, parents=True) # create the repository step 2: create the repo.yaml file that @@ -300,34 +290,32 @@ def generate(self, recipe): # If the recipe provides a package repo, install it as a separate # "recipe" repo in the store with highest precedence. - has_recipe_repo=recipe.spack_repo is not None + has_recipe_repo = recipe.spack_repo is not None if has_recipe_repo: - recipe_dst=repos_path / "recipe" - self._logger.debug( - f"creating the recipe spack repo in {recipe_dst}") + recipe_dst = repos_path / "recipe" + self._logger.debug(f"creating the recipe spack repo in {recipe_dst}") if recipe_dst.exists(): self._logger.debug(f"{recipe_dst} exists ... deleting") shutil.rmtree(recipe_dst) - recipe_pkg_dst=recipe_dst / "packages" + recipe_pkg_dst = recipe_dst / "packages" recipe_pkg_dst.mkdir(mode=0o755, parents=True) with (recipe_dst / "repo.yaml").open("w") as f: f.write(_REPO_YAML.format(namespace="recipe")) - packages_path=recipe.spack_repo / "packages" + packages_path = recipe.spack_repo / "packages" for pkg_path in packages_path.iterdir(): - dst=recipe_pkg_dst / pkg_path.name + dst = recipe_pkg_dst / pkg_path.name if pkg_path.is_dir(): - self._logger.debug(f" installing recipe package { - pkg_path} to {recipe_pkg_dst}") + self._logger.debug(f" installing recipe package {pkg_path} to {recipe_pkg_dst}") install(pkg_path, dst) - repos_yaml_template=jinja_env.get_template("repos.yaml") + repos_yaml_template = jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: - repo_path=recipe.mount / "repos" / "spack_repo" / "alps" - recipe_repo_path=recipe.mount / "repos" / "spack_repo" / "recipe" - package_repos=[ + repo_path = recipe.mount / "repos" / "spack_repo" / "alps" + recipe_repo_path = recipe.mount / "repos" / "spack_repo" / "recipe" + package_repos = [ { "name": pkg_repo["name"], "path": (recipe.mount / "repos" / "spack_repo" / pkg_repo["name"]).as_posix(), @@ -350,12 +338,11 @@ def generate(self, recipe): # repos.yaml config file, recipe packages have precedence. for repo_src in repos: self._logger.debug(f"installing repo {repo_src}") - packages_path=repo_src / "packages" + packages_path = repo_src / "packages" for pkg_path in packages_path.iterdir(): - dst=pkg_dst / pkg_path.name + dst = pkg_dst / pkg_path.name if pkg_path.is_dir() and not dst.exists(): - self._logger.debug(f" installing package { - pkg_path} to {pkg_dst}") + self._logger.debug(f" installing package {pkg_path} to {pkg_dst}") install(pkg_path, dst) elif dst.exists(): self._logger.debug(f" NOT installing package {pkg_path}") @@ -363,22 +350,21 @@ def generate(self, recipe): # Copy all package repos defined in config.yaml to their final repo # locations. for pkg_repo in spack_meta["packages"]: - clone_path=pkg_repo["path"] - name=pkg_repo["name"] - src_path=clone_path / pkg_repo["repo_path"] - dst_path=store_path / "repos" / "spack_repo" / name - self._logger.debug(f"copying repo '{name}' from { - src_path} to {dst_path}") + clone_path = pkg_repo["path"] + name = pkg_repo["name"] + src_path = clone_path / pkg_repo["repo_path"] + dst_path = store_path / "repos" / "spack_repo" / name + self._logger.debug(f"copying repo '{name}' from {src_path} to {dst_path}") if dst_path.exists(): self._logger.debug(f"{dst_path} exists ... deleting") shutil.rmtree(dst_path) install(src_path, dst_path) # --- generate-config subdirectory --- - generate_config_path=self.path / "generate-config" + generate_config_path = self.path / "generate-config" generate_config_path.mkdir(exist_ok=True) - make_config_template=jinja_env.get_template("Makefile.generate-config") + make_config_template = jinja_env.get_template("Makefile.generate-config") with (generate_config_path / "Makefile").open("w") as f: f.write( make_config_template.render( @@ -391,32 +377,29 @@ def generate(self, recipe): # --- modules --- if recipe.with_modules: - modules_path=self.path / "modules" + modules_path = self.path / "modules" modules_path.mkdir(exist_ok=True) with (modules_path / "modules.yaml").open("w") as f: yaml.dump(recipe.modules, f) # --- metadata --- - meta_path=store_path / "meta" + meta_path = store_path / "meta" meta_path.mkdir(exist_ok=True) with (meta_path / "configure.json").open("w") as f: - f.write(json.dumps(self.configuration_meta, - sort_keys=True, indent=2, default=str)) + f.write(json.dumps(self.configuration_meta, sort_keys=True, indent=2, default=str)) f.write("\n") with (meta_path / "env.json.in").open("w") as f: - f.write(json.dumps(self.environment_meta, - sort_keys=True, indent=2, default=str)) + f.write(json.dumps(self.environment_meta, sort_keys=True, indent=2, default=str)) f.write("\n") - meta_recipe_path=meta_path / "recipe" + meta_recipe_path = meta_path / "recipe" if meta_recipe_path.exists(): shutil.rmtree(meta_recipe_path) - install(recipe.path, meta_recipe_path, - ignore=shutil.ignore_patterns(".git")) + install(recipe.path, meta_recipe_path, ignore=shutil.ignore_patterns(".git")) - meta_extra_path=meta_path / "extra" + meta_extra_path = meta_path / "extra" if meta_extra_path.exists(): shutil.rmtree(meta_extra_path) if recipe.user_extra is not None: @@ -425,7 +408,7 @@ def generate(self, recipe): meta_extra_path.mkdir() # --- debug helper --- - debug_template=jinja_env.get_template("stack-debug.sh") + debug_template = jinja_env.get_template("stack-debug.sh") with (self.path / "stack-debug.sh").open("w") as f: f.write( debug_template.render( @@ -439,7 +422,7 @@ def generate(self, recipe): def _git_clone(self, name, repo, commit, path): if not (path / ".git").is_dir(): self._logger.info(f"{name}: clone repository {repo} to {path}") - capture=subprocess.run( + capture = subprocess.run( ["git", "clone", "--filter=tree:0", repo, path], shell=False, stdout=subprocess.PIPE, @@ -454,7 +437,7 @@ def _git_clone(self, name, repo, commit, path): if commit: self._logger.info(f"{name}: fetching {commit}") - capture=subprocess.run( + capture = subprocess.run( ["git", "-C", path, "fetch", "origin", commit], shell=False, stdout=subprocess.PIPE, @@ -465,7 +448,7 @@ def _git_clone(self, name, repo, commit, path): capture.check_returncode() self._logger.info(f"{name}: checking out {commit}") - capture=subprocess.run( + capture = subprocess.run( ["git", "-C", path, "checkout", commit], shell=False, stdout=subprocess.PIPE, @@ -477,7 +460,7 @@ def _git_clone(self, name, repo, commit, path): else: self._logger.info(f"{name}: no commit set") - git_commit=( + git_commit = ( subprocess.run( ["git", "-C", path, "rev-parse", "HEAD"], shell=False, diff --git a/stackinator/etc/Make.inc b/stackinator/etc/Make.inc index 04a2789f..43c3bd63 100644 --- a/stackinator/etc/Make.inc +++ b/stackinator/etc/Make.inc @@ -3,6 +3,17 @@ SPACK ?= spack SPACK_HELPER := $(SPACK) --color=never +# Per-phase banners. Colored on a terminal (MAKE_TERMOUT), plain when piped. +ifeq ($(origin NO_COLOR),undefined) +ifneq ($(MAKE_TERMOUT),) +BANNER_COLOR := \033[1;93m +BANNER_RESET := \033[0m +endif +endif + +# Usage: $(call banner,Human readable phase name) +banner = @printf '\n%b==> [stackinator] %s%b\n' '$(BANNER_COLOR)' '$(1)' '$(BANNER_RESET)' + ifndef STORE $(error STORE should point to a Spack install root) endif diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index 6dd48260..9436bd65 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -713,8 +713,6 @@ def meta_impl(args): args = parser.parse_args() if args.command == "uenv": - print("!!! running meta") meta_impl(args) elif args.command == "view": - print("!!! running view") view_impl(args) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 84dad23e..10cd7517 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -10,6 +10,7 @@ spack-version: # Sanity check: confirm spack works and bootstrap the concretizer. spack-setup: spack-version + $(call banner,Spack setup) @printf "spack arch... " ; \ arch="$$($(SANDBOX) $(SPACK) arch)"; \ printf "%s\n" "$$arch"; \ @@ -27,12 +28,14 @@ spack-setup: spack-version {% if pre_install_hook %} pre-install: spack-setup + $(call banner,Pre-install hook) $(SANDBOX) $(STORE)/pre-install-hook touch pre-install {% endif %} mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} + $(call banner,Build cache / mirror setup) {% if cache %} $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache keys --install --trust --yes-to-all {% if cache.key %} @@ -42,6 +45,7 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} touch mirror-setup concretize: mirror-setup env/spack.yaml + $(call banner,Concretize) $(SANDBOX) $(SPACK) -e $(ENV_ROOT) concretize touch concretize @@ -55,10 +59,12 @@ concretize: mirror-setup env/spack.yaml # Outer-make `-j` no longer governs spack; set build_jobs to control parallelism (or use # GNU make >= 4.4, whose `fifo:` jobserver spack parses correctly). install: concretize + $(call banner,Install packages) MAKEFLAGS= $(SANDBOX) $(SPACK) -e $(ENV_ROOT) install --jobs $(NJOBS) touch install cache-push: install + $(call banner,Push to build cache) {% if push_to_cache and cache.key %} $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache create --rebuild-index --only=package alpscache \ $$($(SANDBOX) $(SPACK_HELPER) -e $(ENV_ROOT) find --format '{name};{/hash};version={version}' \ @@ -70,6 +76,7 @@ cache-push: install touch cache-push cleanup: cache-push + $(call banner,Cleanup) {% if cleanup == "build" %} $(SANDBOX) $(SPACK) gc --yes-to-all --keep-build-dependencies --except-environment $(ENV_ROOT) {% elif cleanup == "runtime" %} @@ -79,6 +86,7 @@ cleanup: cache-push # Generate compiler-config.yaml for use by view and generate-config steps. compiler-config.yaml: cleanup + $(call banner,Generate compiler config) $(SANDBOX) $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ {% if system_gcc %} --system-packages=$(BUILD_ROOT)/config/packages.yaml \ @@ -89,6 +97,7 @@ compiler-config.yaml: cleanup {% for name, config in environments.items() %} {% for view in config.views %} views/{{ view.name }}: install compiler-config.yaml + $(call banner,View: {{ view.name }}) mkdir -p $(dir $@) $(SANDBOX) mkdir -p $(STORE)/env/{{ view.name }} $(SANDBOX) sh -c '$(SPACK) env activate -d $(ENV_ROOT) --with-view {{ view.name }} --sh > $(STORE)/env/{{ view.name }}/activate.sh' @@ -106,31 +115,38 @@ views: {% for name, config in environments.items() %}{% for view in config.views touch views generate-config: install compiler-config.yaml + $(call banner,Generate upstream spack config) $(SANDBOX) $(MAKE) -j1 -C generate-config {% if modules %} modules-done: install generate-config + $(call banner,Generate modules) $(SANDBOX) $(SPACK) -e $(ENV_ROOT) module tcl refresh --upstream-modules --delete-tree --yes-to-all touch modules-done {% endif %} env-meta: generate-config views{% if modules %} modules-done{% endif %} + $(call banner,Generate uenv metadata) $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv \ {% if modules %}--modules {% endif %}\ --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}' \ - --spack-packages='{{ spack_meta.packages_url }},{{ spack_meta.packages_ref }},{{ spack_meta.packages_commit }}' \ +{% for pkg_repo in spack_meta.packages %} + --spack-package-repo='{{ pkg_repo.name }},{{ pkg_repo.url }},{{ pkg_repo.ref }},{{ pkg_repo.commit }}' \ +{% endfor %} $(STORE) touch env-meta {% if post_install_hook %} post-install: env-meta + $(call banner,Post-install hook) $(SANDBOX) $(STORE)/post-install-hook touch post-install {% endif %} store.squashfs: env-meta{% if post_install_hook %} post-install{% endif %}{% if push_to_cache and cache.key %} cache-push{% endif %} + $(call banner,Create squashfs image) $(SANDBOX) find $(STORE)/repos -type d -name __pycache__ -exec rm -r {} + $(SANDBOX) chmod -R a+rX $(STORE) $(SANDBOX) env -u SOURCE_DATE_EPOCH \ From adae8a77f4a18344134912516db1b0cb98983377 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 18 Jun 2026 16:45:44 +0200 Subject: [PATCH 13/15] view generation output is easier to read --- stackinator/etc/envvars.py | 17 +++-------------- stackinator/templates/Makefile | 10 +++------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index 9436bd65..c3ab8a04 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -476,11 +476,6 @@ def read_activation_script(filename: str, env: Optional[EnvVarSet] = None) -> En def view_impl(args): - print( - f"parsing view {args.root}\n compilers {args.compilers}\n prefix_paths '{args.prefix_paths}'\n \ - build_path '{args.build_path}'" - ) - if not os.path.isdir(args.root): print(f"error - environment root path {args.root} does not exist") exit(1) @@ -523,6 +518,7 @@ def view_impl(args): c = e["extra_attributes"].get("compilers") if not c: continue + print(f"compiler symlinks: creating for {e.get('prefix', pkg_name)}") for role, path in c.items(): if path is None: continue @@ -532,16 +528,11 @@ def view_impl(args): else: link_name = os.path.basename(src) dst = os.path.join(bin_path, link_name) - print(f"creating compiler symlink: {src} -> {dst}") if os.path.exists(dst): - print(f" first removing {dst}") os.remove(dst) os.symlink(src, dst) if args.prefix_paths: - # get the root path of the env - print(f"prefix_paths: searching in {root_path}") - for p in args.prefix_paths.split(","): name, value = p.split("=") paths = [] @@ -550,11 +541,9 @@ def view_impl(args): if os.path.isdir(test_path): paths.append(test_path) - print(f"{name}:") - for p in paths: - print(f" {p}") - if len(paths) > 0: + print(f"{name}: {' '.join(paths)}") + if name in envvars.lists: ld_paths = envvars.lists[name].paths final_paths = [p for p in paths if p not in ld_paths] diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10cd7517..8785696f 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -93,12 +93,12 @@ compiler-config.yaml: cleanup {% endif %} $(BUILD_ROOT)/compiler-config.yaml{% if compiler_names %} {{ compiler_names | join(' ') }}{% endif %} -# Generate activate.sh and env.json for each environment view. +# Generate activate.sh and env.json for each environment view. All views are +# built in a single target so their output is not interleaved under make -j. +views: install compiler-config.yaml {% for name, config in environments.items() %} {% for view in config.views %} -views/{{ view.name }}: install compiler-config.yaml $(call banner,View: {{ view.name }}) - mkdir -p $(dir $@) $(SANDBOX) mkdir -p $(STORE)/env/{{ view.name }} $(SANDBOX) sh -c '$(SPACK) env activate -d $(ENV_ROOT) --with-view {{ view.name }} --sh > $(STORE)/env/{{ view.name }}/activate.sh' $(SANDBOX) $(BUILD_ROOT)/envvars.py view \ @@ -106,12 +106,8 @@ views/{{ view.name }}: install compiler-config.yaml --prefix_paths="{{ view.extra.prefix_string }}" \ $(STORE)/env/{{ view.name }} \ $(BUILD_ROOT) - touch $@ - {% endfor %} {% endfor %} -views: {% for name, config in environments.items() %}{% for view in config.views %}views/{{ view.name }} {% endfor %}{% endfor %} - touch views generate-config: install compiler-config.yaml From 42819b52abea5e88e1738d78af05d0ada47dbc39 Mon Sep 17 00:00:00 2001 From: bcumming Date: Thu, 18 Jun 2026 17:46:39 +0200 Subject: [PATCH 14/15] make sandboxed shell commands easier to read --- stackinator/builder.py | 13 ++++++++++++ stackinator/etc/Make.inc | 4 ++++ stackinator/templates/Make.user | 21 ++++++------------- .../templates/Makefile.generate-config | 4 +--- stackinator/templates/sandbox | 18 ++++++++++++++++ 5 files changed, 42 insertions(+), 18 deletions(-) create mode 100644 stackinator/templates/sandbox diff --git a/stackinator/builder.py b/stackinator/builder.py index a061e0aa..169509cd 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -212,6 +212,19 @@ def generate(self, recipe): ) f.write("\n") + # --- Write the sandbox wrapper (binds baked in, self-labelling) --- + sandbox_template = jinja_env.get_template("sandbox") + sandbox_dst = self.path / "sandbox" + with sandbox_dst.open("w") as f: + f.write( + sandbox_template.render( + build_path=self.path, + store=recipe.mount, + no_bwrap=recipe.no_bwrap, + ) + ) + os.chmod(sandbox_dst, os.stat(sandbox_dst).st_mode | stat.S_IEXEC) + # --- Copy static files from etc/ --- etc_path = self.root / "etc" for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py", "compiler-config.py"]: diff --git a/stackinator/etc/Make.inc b/stackinator/etc/Make.inc index 43c3bd63..cc7f9c39 100644 --- a/stackinator/etc/Make.inc +++ b/stackinator/etc/Make.inc @@ -1,5 +1,9 @@ # vi: filetype=make +# Suppress make's recipe echo. Sandboxed commands self-label via the sandbox +# wrapper, and build phases are announced by the banner macro below. +.SILENT: + SPACK ?= spack SPACK_HELPER := $(SPACK) --color=never diff --git a/stackinator/templates/Make.user b/stackinator/templates/Make.user index 55665638..833cd441 100644 --- a/stackinator/templates/Make.user +++ b/stackinator/templates/Make.user @@ -18,21 +18,12 @@ SPACK_HELPER := $(SPACK) --color=never # The Spack installation root. STORE := {{ store }} -# When already building inside a sandbox, use `SANDBOX :=` (empty string) -# Without a sandbox, make sure to hide sensitive data such as ~/.ssh through bubblewrap. -# Also bind the directories `./tmp -> /tmp` and `./store -> $(STORE)`, so that -# builds and installs happen inside the current directory. For speed, either -# put the project itself in-memory, or use a flag like --bind /dev/shm/store -# $(STORE). Use `bwrap-mutable-root.sh` in case you need to create a new -# directory at the root /. -{% if no_bwrap %} -SANDBOX := -{% else %} -SANDBOX := $(BUILD_ROOT)/bwrap-mutable-root.sh $\ - --tmpfs ~ $\ - --bind $(BUILD_ROOT)/tmp /tmp $\ - --bind $(BUILD_ROOT)/store $(STORE) -{% endif %} +# Wrapper that runs a command inside the bubblewrap sandbox (binding ./tmp -> +# /tmp and ./store -> $(STORE) and hiding ~). It prints a compact +# `$(BUILD_ROOT)/sandbox ` label so the build output is readable. The bind +# mounts live in the wrapper itself; see $(BUILD_ROOT)/sandbox. When no_bwrap is +# set the wrapper just runs the command directly. +SANDBOX := $(BUILD_ROOT)/sandbox # Makes sure that make -Orecurse continues to print in color. export SPACK_COLOR := always diff --git a/stackinator/templates/Makefile.generate-config b/stackinator/templates/Makefile.generate-config index 76e11d55..11cd91bb 100644 --- a/stackinator/templates/Makefile.generate-config +++ b/stackinator/templates/Makefile.generate-config @@ -12,9 +12,7 @@ $(CONFIG_DIR)/upstreams.yaml: # Generate packages.yaml with the spack-built compiler externals so that # downstream Spack users can find and use the compilers without a compiler.yaml. $(CONFIG_DIR)/packages.yaml: $(CONFIG_DIR)/upstreams.yaml - $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ - $(CONFIG_DIR)/packages.yaml \ - {{ compiler_names | join(' ') }} + $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py $(CONFIG_DIR)/packages.yaml {{ compiler_names | join(' ') }} $(CONFIG_DIR)/repos.yaml: $(CONFIG_DIR)/packages.yaml install -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml diff --git a/stackinator/templates/sandbox b/stackinator/templates/sandbox new file mode 100644 index 00000000..65babbf7 --- /dev/null +++ b/stackinator/templates/sandbox @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Pretty-print the logical command, then run it under the bwrap sandbox. +# The label goes to stderr so that recipes which capture the sandbox output in +# command substitution (e.g. `x="$(SANDBOX spack --version)"`) are unaffected. +if [ -z "${NO_COLOR:-}" ] && [ -t 2 ]; then + printf '\033[38;5;45m%s\033[0m %s\n' "$0" "$*" >&2 +else + printf '%s %s\n' "$0" "$*" >&2 +fi +{% if no_bwrap %} +exec "$@" +{% else %} +exec {{ build_path }}/bwrap-mutable-root.sh \ + --tmpfs ~ \ + --bind {{ build_path }}/tmp /tmp \ + --bind {{ build_path }}/store {{ store }} \ + "$@" +{% endif %} From aaedf1c81c17bf4a0de686a9be9f012424f4c7fe Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 19 Jun 2026 17:26:36 +0200 Subject: [PATCH 15/15] fix cache push and module generation - still need to generate upstreams.yaml correctly --- stackinator/builder.py | 9 +- stackinator/etc/compiler-config.py | 7 ++ stackinator/etc/envvars.py | 15 +++ stackinator/recipe.py | 17 ++- stackinator/templates/Makefile | 113 +++++++++++------- .../templates/Makefile.generate-config | 38 ++++-- 6 files changed, 144 insertions(+), 55 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 169509cd..78349a94 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -189,7 +189,7 @@ def generate(self, recipe): environments=recipe.environments, compiler_names=recipe.compiler_names, gpg_keys=recipe.mirrors.gpg_key_paths(config_path), - cache=recipe.build_cache_mirror, + buildcache=recipe.build_cache_mirror, buildcache_push=recipe.push_to_build_cache, exclude_from_cache=["nvhpc", "cuda", "perl"], has_views=has_views, @@ -256,7 +256,7 @@ def generate(self, recipe): # - the system packages.yaml with gcc removed # - plus additional packages provided by the recipe with (config_path / "packages.yaml").open("w") as f: - f.write(yaml.dump(recipe.packages)) + f.write(yaml.dump(recipe.packages["build"])) config_yaml = {"config": {"install_tree": {"root": str(recipe.mount)}}} with (config_path / "config.yaml").open("w") as f: @@ -384,9 +384,12 @@ def generate(self, recipe): modules=recipe.with_modules, build_path=self.path.as_posix(), compiler_names=recipe.compiler_names, + system_gcc=recipe.system_gcc, ) ) f.write("\n") + with (generate_config_path / "packages.yaml").open("w") as f: + f.write(yaml.dump(recipe.packages["install"])) # --- modules --- if recipe.with_modules: @@ -394,6 +397,8 @@ def generate(self, recipe): modules_path.mkdir(exist_ok=True) with (modules_path / "modules.yaml").open("w") as f: yaml.dump(recipe.modules, f) + with (modules_path / "packages.yaml").open("w") as f: + f.write(yaml.dump(recipe.packages["install"])) # --- metadata --- meta_path = store_path / "meta" diff --git a/stackinator/etc/compiler-config.py b/stackinator/etc/compiler-config.py index c55a0d08..a6cb44df 100644 --- a/stackinator/etc/compiler-config.py +++ b/stackinator/etc/compiler-config.py @@ -59,6 +59,13 @@ def build_compiler_packages(compiler_names): externals = [] for spec in specs: + # Skip externals such as the system gcc. It is registered in the + # cluster packages.yaml and gets pulled into the DB as a bootstrap + # dependency, but only compilers actually built into this environment + # should be surfaced here. The system compiler is included separately, + # and only when selected, via --system-packages. + if spec.external: + continue prefix = str(spec.prefix) bins = find_compiler_bins(prefix, name) if not bins: diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index c3ab8a04..8994628e 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -511,7 +511,15 @@ def view_impl(args): with open(args.compilers, "r") as file: data = yaml.safe_load(file) + # Restrict the symlinked compilers to those wired to this view's + # environment (environments.yaml: compiler). When None, link them all. + allowed = None + if args.compiler_names: + allowed = {n for n in args.compiler_names.split(",") if n} + for pkg_name, pkg_data in data["packages"].items(): + if allowed is not None and pkg_name not in allowed: + continue for e in pkg_data["externals"]: if "extra_attributes" not in e: continue @@ -676,6 +684,13 @@ def meta_impl(args): ) # only add compilers if this argument is passed view_parser.add_argument("--compilers", help="path of the packages.yaml file", type=str, default=None) + view_parser.add_argument( + "--compiler-names", + help="comma-separated compiler package names to symlink into the view; " + "restricts --compilers to the compilers wired to this environment", + type=str, + default=None, + ) uenv_parser = subparsers.add_parser( "uenv", diff --git a/stackinator/recipe.py b/stackinator/recipe.py index a174984f..374da354 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -144,9 +144,17 @@ def __init__(self, args): # note that the order that package sets are specified in is significant. # arguments to the right have higher precedence. - # Global packages.yaml: system + network + recipe packages. - # gcc is included here (unlike v2 which isolated it for the bootstrap step). - self.packages = {"packages": system_packages | network_packages | recipe_packages} + # + # build_packages: system + network + recipe packages. + # - system gcc is included here because needed to build the software + # global_packages: system + network + recipe packages. + # - system gcc is only included if building with system gcc + build_packages = system_packages | network_packages | recipe_packages + install_packages = system_packages | network_packages | recipe_packages + if not self.use_system_gcc: + del install_packages["gcc"] + + self.packages = {"build": {"packages": build_packages}, "install": {"packages": install_packages}} # required environments.yaml file environments_path = self.path / "environments.yaml" @@ -505,6 +513,9 @@ def generate_compiler_specs(self, raw): self.compilers = compilers + # will the uenv use the system gcc instead of bootstrapping gcc + self.use_system_gcc = (gcc_version == "system") + @property def system_gcc(self): return self.compilers.get("gcc", {}).get("system", False) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 8785696f..8fe59def 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -5,20 +5,22 @@ all: store.squashfs +# Keep track of what Spack version was used. spack-version: $(SANDBOX) $(SPACK) --version > $@ # Sanity check: confirm spack works and bootstrap the concretizer. +# +# spack-setup: spack-version - $(call banner,Spack setup) @printf "spack arch... " ; \ arch="$$($(SANDBOX) $(SPACK) arch)"; \ printf "%s\n" "$$arch"; \ printf "spack version... "; \ version="$$($(SANDBOX) $(SPACK) --version)"; \ printf "%s\n" "$$version"; \ - printf "checking if spack concretizer works... "; \ - $(SANDBOX) $(SPACK_HELPER) -d spec zlib > $(BUILD_ROOT)/spack-bootstrap-output 2>&1; \ + printf "bootstrapping spack... "; \ + $(SANDBOX) $(SPACK_HELPER) bootstrap now > $(BUILD_ROOT)/spack-bootstrap-output 2>&1; \ if [ "$$?" != "0" ]; then \ printf " failed, see %s\n" $(BUILD_ROOT)/spack-bootstrap-output; \ exit 1; \ @@ -26,47 +28,49 @@ spack-setup: spack-version printf " success\n"; \ touch spack-setup -{% if pre_install_hook %} pre-install: spack-setup - $(call banner,Pre-install hook) + $(call banner,pre-install hook) +{% if pre_install_hook %} $(SANDBOX) $(STORE)/pre-install-hook - touch pre-install +{% else %} + echo "no pre install hook" {% endif %} + touch pre-install -mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - - $(call banner,Build cache / mirror setup) -{% if cache %} - $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache keys --install --trust --yes-to-all -{% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust --yes-to-all {{ cache.key }} -{% endif %} -{% endif %} +mirror-setup: spack-setup pre-install + + $(call banner,build cache / mirror setup) + {% if buildcache %} + @echo "Pulling and trusting keys from configured buildcaches." + $(SANDBOX) $(SPACK) buildcache keys --install --trust --yes-to-all + {% endif %} + @echo "Adding mirror gpg keys." + {% for key_path in gpg_keys %} + $(SANDBOX) $(SPACK) gpg trust --yes-to-all {{ key_path }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list touch mirror-setup concretize: mirror-setup env/spack.yaml - $(call banner,Concretize) - $(SANDBOX) $(SPACK) -e $(ENV_ROOT) concretize + $(call banner,concretize) + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) concretize #--force touch concretize # Clear MAKEFLAGS for the install. When run with `make -j`, GNU make advertises a # jobserver in MAKEFLAGS. GNU make < 4.4 uses the legacy fd form (`--jobserver-auth=R,W`) # and closes those fds before running a non-recursive recipe, so they are invalid in the -# spack process. Spack's jobserver probe validates them with `fcntl(fd, F_GETFD) != -1`, -# a C idiom that in Python *raises* OSError(EBADF) instead of returning -1 - so the -# install aborts with `[Errno 9] Bad file descriptor` (independent of Python version). +# spack process - leading to bad file descriptor crashes. # Hiding MAKEFLAGS makes spack create its own FIFO jobserver sized by `config:build_jobs`. -# Outer-make `-j` no longer governs spack; set build_jobs to control parallelism (or use -# GNU make >= 4.4, whose `fifo:` jobserver spack parses correctly). install: concretize - $(call banner,Install packages) + $(call banner,install packages) MAKEFLAGS= $(SANDBOX) $(SPACK) -e $(ENV_ROOT) install --jobs $(NJOBS) touch install cache-push: install - $(call banner,Push to build cache) -{% if push_to_cache and cache.key %} - $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache create --rebuild-index --only=package alpscache \ + $(call banner,push to build cache) +{% if buildcache_push %} + $(SANDBOX) $(SPACK) -e $(ENV_ROOT) buildcache create --only=package {{ buildcache }} \ $$($(SANDBOX) $(SPACK_HELPER) -e $(ENV_ROOT) find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ @@ -76,17 +80,19 @@ cache-push: install touch cache-push cleanup: cache-push - $(call banner,Cleanup) + $(call banner,garbage collection) {% if cleanup == "build" %} $(SANDBOX) $(SPACK) gc --yes-to-all --keep-build-dependencies --except-environment $(ENV_ROOT) {% elif cleanup == "runtime" %} $(SANDBOX) $(SPACK) gc --yes-to-all --except-environment $(ENV_ROOT) +{% else %} + echo "no garbage collection requested" {% endif %} touch cleanup # Generate compiler-config.yaml for use by view and generate-config steps. compiler-config.yaml: cleanup - $(call banner,Generate compiler config) + $(call banner,generate compiler config) $(SANDBOX) $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ {% if system_gcc %} --system-packages=$(BUILD_ROOT)/config/packages.yaml \ @@ -98,11 +104,11 @@ compiler-config.yaml: cleanup views: install compiler-config.yaml {% for name, config in environments.items() %} {% for view in config.views %} - $(call banner,View: {{ view.name }}) + $(call banner,view: {{ view.name }}) $(SANDBOX) mkdir -p $(STORE)/env/{{ view.name }} $(SANDBOX) sh -c '$(SPACK) env activate -d $(ENV_ROOT) --with-view {{ view.name }} --sh > $(STORE)/env/{{ view.name }}/activate.sh' $(SANDBOX) $(BUILD_ROOT)/envvars.py view \ - {% if view.extra.add_compilers %}--compilers=$(BUILD_ROOT)/compiler-config.yaml {% endif %}\ + {% if view.extra.add_compilers %}--compilers=$(BUILD_ROOT)/compiler-config.yaml --compiler-names={{ config.compiler | join(',') }} {% endif %}\ --prefix_paths="{{ view.extra.prefix_string }}" \ $(STORE)/env/{{ view.name }} \ $(BUILD_ROOT) @@ -111,19 +117,23 @@ views: install compiler-config.yaml touch views generate-config: install compiler-config.yaml - $(call banner,Generate upstream spack config) + $(call banner,generate upstream spack config) $(SANDBOX) $(MAKE) -j1 -C generate-config +modules-done: generate-config + $(call banner,generate modules) {% if modules %} -modules-done: install generate-config - $(call banner,Generate modules) - $(SANDBOX) $(SPACK) -e $(ENV_ROOT) module tcl refresh --upstream-modules --delete-tree --yes-to-all - touch modules-done + {% for module_type in module_types %} + $(SANDBOX) $(SPACK) -C $(BUILD_ROOT)/modules module {{ module_type }} refresh --upstream-modules --delete-tree --yes-to-all + {% endfor %} +{% else %} + echo "no modules in this uenv" {% endif %} + touch modules-done -env-meta: generate-config views{% if modules %} modules-done{% endif %} +env-meta: generate-config views modules-done - $(call banner,Generate uenv metadata) + $(call banner,generate uenv metadata) $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv \ {% if modules %}--modules {% endif %}\ --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}' \ @@ -133,16 +143,37 @@ env-meta: generate-config views{% if modules %} modules-done{% endif %} $(STORE) touch env-meta -{% if post_install_hook %} post-install: env-meta - $(call banner,Post-install hook) + $(call banner,post-install hook) +{% if post_install_hook %} $(SANDBOX) $(STORE)/post-install-hook +{% else %} + echo "no post install hook" +{% endif %} touch post-install + +# Force push all built packages to the build cache +cache-force: mirror-setup +{% if buildcache_push %} + $(warning ================================================================================) + $(warning Generate the config in order to force push partially built compiler environments) + $(warning if this step is performed with partially built compiler envs, you will) + $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) + $(warning ================================================================================) + $(SANDBOX) $(MAKE) -C generate-config + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --only=package {{ buildcache_push }} \ + $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ + | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ + | grep -v -E 'version=git\.'\ + | cut -d ';' -f2)\ + | grep -Ev '^==> Fetching|^gpg:' +{% else %} + $(warning "pushing to the build cache is not enabled. See the documentation on how to add a key: https://eth-cscs.github.io/stackinator/build-caches/") {% endif %} -store.squashfs: env-meta{% if post_install_hook %} post-install{% endif %}{% if push_to_cache and cache.key %} cache-push{% endif %} +store.squashfs: env-meta post-install cache-push - $(call banner,Create squashfs image) + $(call banner,create squashfs image) $(SANDBOX) find $(STORE)/repos -type d -name __pycache__ -exec rm -r {} + $(SANDBOX) chmod -R a+rX $(STORE) $(SANDBOX) env -u SOURCE_DATE_EPOCH \ diff --git a/stackinator/templates/Makefile.generate-config b/stackinator/templates/Makefile.generate-config index 11cd91bb..b23f1394 100644 --- a/stackinator/templates/Makefile.generate-config +++ b/stackinator/templates/Makefile.generate-config @@ -1,27 +1,47 @@ include ../Make.user CONFIG_DIR = $(STORE)/config +MODULE_DIR = $(BUILD_ROOT)/modules all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml{% if modules %} $(MODULE_DIR)/upstreams.yaml{% endif %} +# TODO: the upstreams.yaml files are not getting created in either MODULE_DIR or CONFIG_DIR... whats up with that. $(CONFIG_DIR)/upstreams.yaml: + $(call banner,generate $(CONFIG_DIR)/upstreams.yaml) mkdir -p $(CONFIG_DIR) $(SPACK) -e $(ENV_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) +{% if modules %} +$(MODULE_DIR)/upstreams.yaml: + $(call banner,generate $(MODULE_DIR)/upstreams.yaml) + $(SPACK) -e $(ENV_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) + +{% endif %} + # Generate packages.yaml with the spack-built compiler externals so that # downstream Spack users can find and use the compilers without a compiler.yaml. -$(CONFIG_DIR)/packages.yaml: $(CONFIG_DIR)/upstreams.yaml - $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py $(CONFIG_DIR)/packages.yaml {{ compiler_names | join(' ') }} +# When the recipe uses the system gcc, also pull it in from the build packages.yaml. +# +# TODO: we have replaced this with a simple copy of the packages.yaml used to build +# the software stack +# - without gcc when gcc was built in the uenv +# - with system gcc when the system gcc was used +# maybe this will be sufficient with spack 1.2 for downstream consumers: explore with +# spack-uenv to see whether we still need to explicitly add gcc to the uenv config +# in order for spack to find it. +# +#$(CONFIG_DIR)/packages.yaml: $(CONFIG_DIR)/upstreams.yaml +# $(SPACK) -e $(ENV_ROOT) python $(BUILD_ROOT)/compiler-config.py \ +#{% if system_gcc %} +# --system-packages=$(BUILD_ROOT)/config/packages.yaml \ +#{% endif %} +# $(CONFIG_DIR)/packages.yaml{% if compiler_names %} {{ compiler_names | join(' ') }}{% endif %} + +$(CONFIG_DIR)/packages.yaml: + install -m 644 $(BUILD_ROOT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml: $(CONFIG_DIR)/packages.yaml install -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml -{% if modules %} -MODULE_DIR = $(BUILD_ROOT)/modules - -$(MODULE_DIR)/upstreams.yaml: - $(SPACK) -e $(ENV_ROOT) config --scope=user add upstreams:system:install_tree:$(STORE) - -{% endif %} include ../Make.inc