diff --git a/.github/actions/conformance/action.yml b/.github/actions/conformance/action.yml index 60270d3..4a3756f 100644 --- a/.github/actions/conformance/action.yml +++ b/.github/actions/conformance/action.yml @@ -1,13 +1,15 @@ -# Checks a codec against fspm-TF's conformance files, in a codec's own CI, once the codec has run: +# Checks a codec against encoding3d's conformance files, in a codec's own CI, once the codec has run: # -# - uses: ./.github/actions/conformance (in this repository; elsewhere openalea/fspm-TF/.github/actions/conformance@main) +# - uses: ./.github/actions/conformance (in this repository; elsewhere openalea/encoding3d/.github/actions/conformance@main) # with: # dumps: out/dumps # what the codec read: one JSON dump per conformance GLB, x.json for x.expected.json -# written: out/written # what the codec wrote: the conformance scenes, as .glb files (optional) +# written: out/written # what the codec wrote: the conformance scenes, as .glb files (optional), and +# # archives (.fspm.zip) when the codec packs +# instances: "true" # "false" for a codec reading 0.3 only: no instances (0.4) and archives # -# Reading passes when every dump equals its expected values; writing, when every written file passes fspm-tf validate -# and Khronos' glTF-Validator with no error. -name: fspm-TF conformance +# Reading passes when every dump equals its expected values (the dumps' layout: conformance/README.md); writing, when +# every written file passes encoding3d validate and Khronos' glTF-Validator with no error. +name: encoding3d conformance description: Compare a codec's reading of the conformance files with their expected values, and validate what it wrote inputs: dumps: @@ -17,8 +19,12 @@ inputs: description: the folder of the .glb files the codec wrote for the conformance scenes (empty to skip) required: false default: "" - fspm-tf: - description: the path of an fspm-TF checkout (its conformance folder and Python package) + instances: + description: whether the codec reads 0.4's instances and archives (their dumps compared); "false" skips them + required: false + default: "true" + encoding3d: + description: the path of an encoding3d checkout (its conformance folder and Python package) required: false default: "." gltf-validator: @@ -31,20 +37,23 @@ runs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install openalea.fspm_tf + - name: Install openalea.encoding3d shell: bash - run: python -m pip install numpy jsonschema "${{ inputs.fspm-tf }}" + run: python -m pip install numpy jsonschema "${{ inputs.encoding3d }}" - name: Reading shell: bash - run: fspm-tf compare-all "${{ inputs.dumps }}" "${{ inputs.fspm-tf }}/conformance" - - name: Writing, fspm-TF validator + run: encoding3d compare-all "${{ inputs.dumps }}" "${{ inputs.encoding3d }}/conformance" ${{ inputs.instances == 'false' && '--skip-instances' || '' }} + - name: Writing, encoding3d validator if: inputs.written != '' shell: bash - run: find "${{ inputs.written }}" -name "*.glb" -print0 | xargs -0 fspm-tf validate + run: find "${{ inputs.written }}" \( -name "*.glb" -o -name "*.fspm.zip" \) -print0 | xargs -0 encoding3d validate - name: Writing, Khronos' glTF-Validator if: inputs.written != '' && runner.os == 'Linux' shell: bash run: | curl -sSL -o validator.tar.xz "https://github.com/KhronosGroup/glTF-Validator/releases/download/${{ inputs.gltf-validator }}/gltf_validator-${{ inputs.gltf-validator }}-linux64.tar.xz" tar xf validator.tar.xz gltf_validator - find "${{ inputs.written }}" -name "*.glb" -print0 | xargs -0 -n1 ./gltf_validator + find "${{ inputs.written }}" -name "*.glb" -print0 | xargs -0 -n1 ./gltf_validator || { + # each file's issues are in its report beside it: print the errors + find "${{ inputs.written }}" -name "*.report.json" -exec python -c "import json, sys; report = json.load(open(sys.argv[1])); [print(sys.argv[1], m['code'], m.get('pointer', ''), m['message']) for m in report['issues']['messages'] if m['severity'] == 0]" {} \; + exit 1; } diff --git a/.github/workflows/conda-build.yml b/.github/workflows/conda-build.yml index 6714743..2f014e1 100644 --- a/.github/workflows/conda-build.yml +++ b/.github/workflows/conda-build.yml @@ -22,3 +22,14 @@ jobs: secrets: anaconda_token: ${{ secrets.ANACONDA_TOKEN }} + ci-ok: + name: CI OK + needs: build + if: always() # run even if build failed/cancelled, otherwise it'd be "skipped" (= passes!) + runs-on: ubuntu-latest + steps: + - name: Check all OpenAlea CI jobs succeeded + run: | + echo "build result: ${{ needs.build.result }}" + [ "${{ needs.build.result }}" = "success" ] + diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index dae2e39..073b4ef 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,4 +1,4 @@ -# The conformance files are valid glTF for every viewer (Khronos' glTF-Validator) and valid fspm-TF (our validator) +# The conformance files are valid glTF for every viewer (Khronos' glTF-Validator) and valid encoding3d (our validator) name: Conformance on: @@ -17,22 +17,24 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install openalea.fspm_tf (its dependencies, conda-only in pyproject.toml, given here), and VTK for the ParaView reader's tests - run: python -m pip install numpy jsonschema pytest vtk . + - name: Install openalea.encoding3d (its dependencies, conda-only in pyproject.toml, given here), and VTK for the ParaView reader's tests + run: python -m pip install numpy jsonschema scipy pytest pyzmq vtk . - name: Tests, the ParaView reader's conversion included run: pytest bindings/python/test -v - name: The committed files are the generated ones run: | python conformance/generate.py - git diff --exit-code -- conformance - - name: fspm-TF validator - run: fspm-tf validate conformance/*.glb conformance/growing_sequence/scene.fspm.json + # the evaluated values' last digits follow the platform's BLAS: test_conformance compares them within the + # protocol's 1e-12; every other file must be what the generator writes, byte for byte + git diff --exit-code -- conformance ':(exclude)conformance/instances/*.evaluated.json' + - name: encoding3d validator + run: encoding3d validate conformance/*.glb conformance/growing_sequence/scene.fspm.json conformance/instances/scene.fspm.json conformance/instances/embedded.glb conformance/archives/*.fspm.zip - name: Khronos glTF-Validator run: | curl -sSL -o validator.tar.xz "https://github.com/KhronosGroup/glTF-Validator/releases/download/${GLTF_VALIDATOR}/gltf_validator-${GLTF_VALIDATOR}-linux64.tar.xz" tar xf validator.tar.xz gltf_validator - for file in conformance/*.glb conformance/growing_sequence/*.glb; do - ./gltf_validator "$file" || exit 1 + for file in conformance/*.glb conformance/growing_sequence/*.glb conformance/instances/*.glb conformance/instances/parts/*.glb; do + ./gltf_validator "$file" || { python -c "import json, sys; report = json.load(open(sys.argv[1])); [print(sys.argv[1], m['code'], m.get('pointer', ''), m['message']) for m in report['issues']['messages'] if m['severity'] == 0]" "${file}.report.json"; exit 1; } done docs: @@ -42,10 +44,14 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install openalea.fspm_tf and the documentation's tools - run: python -m pip install numpy jsonschema sphinx pydata-sphinx-theme myst-parser sphinx-favicon sphinx-copybutton . + - name: Install openalea.encoding3d and the documentation's tools + run: | + sudo apt-get install -y pandoc + python -m pip install numpy jsonschema pytest sphinx ".[doc]" + - name: The notebooks run + run: python -m pytest --nbmake doc/notebooks tutorials/01_encoding3d_alone - name: Build the documentation, warnings as errors - run: python -m sphinx -b html docs docs/_build -W --keep-going + run: python -m sphinx -b html doc doc/_build -W --keep-going # The reference codec through the action other codecs use: its dumps of the conformance files, and the files it writes python-codec-conformance: @@ -57,7 +63,7 @@ jobs: python-version: "3.12" - name: Dump and write the conformance scenes with the Python codec run: | - python -m pip install numpy jsonschema . + python -m pip install numpy jsonschema scipy . python bindings/python/conformance_run.py conformance out - uses: ./.github/actions/conformance with: diff --git a/.github/workflows/cpp-codec.yml b/.github/workflows/cpp-codec.yml new file mode 100644 index 0000000..7023473 --- /dev/null +++ b/.github/workflows/cpp-codec.yml @@ -0,0 +1,51 @@ +# The C++ codec (bindings/cpp): its tests on Linux, macOS and Windows; the conformance check other codecs run (its +# dumps of the conformance files compared with their expected values, the files it writes validated); the coupling +# conformance sessions. No platform: CPlantBox's adapter (bindings/cpp/cplantbox), a proposal for CPlantBox, is tested +# by hand (bindings/cpp/cplantbox/README.md). OpenAlea's conda CI builds the Python package only. +name: C++ codec + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - name: Configure (nlohmann_json fetched) + run: cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release + - name: Build + run: cmake --build build --config Release + - name: Tests + run: ctest --test-dir build -C Release --output-on-failure + + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Dump and write the conformance scenes with the C++ codec + run: | + cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build --target encoding3d_conformance_run + ./build/encoding3d_conformance_run conformance out + - uses: ./.github/actions/conformance + with: + dumps: out/dumps + written: out/written + # the coupling conformance sessions (specification §9, §11): the C++ participant, over ZeroMQ (libzmq) and a folder + - name: The coupling conformance sessions + run: | + sudo apt-get install -y -q libzmq3-dev + cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build --target encoding3d_coupling_conformance + python -m pip install numpy jsonschema pyzmq . + encoding3d couple-conformance conformance/coupling --participant "$PWD/build/encoding3d_coupling_conformance" + diff --git a/.github/workflows/enforce-source-branch.yml b/.github/workflows/enforce-source-branch.yml new file mode 100644 index 0000000..fc28efd --- /dev/null +++ b/.github/workflows/enforce-source-branch.yml @@ -0,0 +1,19 @@ +name: Enforce source branch + +on: + pull_request: + branches: + - main + +jobs: + check-source-branch: + runs-on: ubuntu-latest + steps: + - name: Check PR source branch + run: | + if [ "${{ github.head_ref }}" != "develop" ]; then + echo "❌ PRs to main must originate from the 'develop' branch." + echo " This PR is from: ${{ github.head_ref }}" + exit 1 + fi + echo "✅ Source branch is 'develop'. OK." diff --git a/.github/workflows/java-codec.yml b/.github/workflows/java-codec.yml new file mode 100644 index 0000000..f9e661e --- /dev/null +++ b/.github/workflows/java-codec.yml @@ -0,0 +1,53 @@ +# The Java codec (bindings/java): its tests on Linux, macOS and Windows; the conformance check other codecs run; the +# coupling conformance sessions. No platform: GroIMP's adapter and plugin (bindings/java/groimp, groimp-plugin), +# proposals for GroIMP, are tested by hand (their README.md). OpenAlea's conda CI builds the Python package only. +name: Java codec + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + cache: maven + - name: Tests, the conformance files included + run: mvn -B -f bindings/java/pom.xml verify + + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + cache: maven + - name: Dump and write the conformance scenes with the Java codec + run: | + mvn -B -q -f bindings/java/pom.xml package -DskipTests dependency:copy-dependencies -DincludeScope=runtime + java -cp "bindings/java/target/encoding3d-0.1.0.jar:bindings/java/target/dependency/*" org.openalea.encoding3d.ConformanceRun conformance out + - uses: ./.github/actions/conformance + with: + dumps: out/dumps + written: out/written + # the coupling conformance sessions (specification §9, §11): the Java participant, over ZeroMQ (JeroMQ) and a folder + - name: The coupling conformance sessions + run: | + python -m pip install pyzmq + encoding3d couple-conformance conformance/coupling --participant \ + "java -cp $PWD/bindings/java/target/encoding3d-0.1.0.jar:$PWD/bindings/java/target/dependency/* org.openalea.encoding3d.coupling.ConformanceParticipant" + diff --git a/.github/workflows/julia-codec.yml b/.github/workflows/julia-codec.yml new file mode 100644 index 0000000..99ed09e --- /dev/null +++ b/.github/workflows/julia-codec.yml @@ -0,0 +1,86 @@ +# The Julia codec (bindings/julia/encoding3d): its tests, on the oldest supported Julia and the latest release, and the +# conformance check other codecs run (its dumps of the conformance files compared with their expected values, the files +# it writes validated by encoding3d validate and Khronos' glTF-Validator); its coupling participant. No platform: the +# package extensions for VirtualPlantLab (PlantGraphs, PlantGeomTurtle) and MTG.jl, proposals for those platforms, are +# tested by hand in their own projects (test/plantgraphs, test/multiscaletreegraph; bindings/julia/README.md). OpenAlea's +# conda CI builds the Python package only. +name: Julia codec + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + name: Julia ${{ matrix.julia-version }}, ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + julia-version: ["1.10", "1"] + os: [ubuntu-latest] + include: + - julia-version: "1" + os: windows-latest + - julia-version: "1" + os: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.julia-version }} + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + with: + project: bindings/julia/encoding3d + - uses: julia-actions/julia-runtest@v1 + with: + project: bindings/julia/encoding3d + + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: "1" + - uses: julia-actions/cache@v2 + - name: Dump and write the conformance scenes with the Julia codec + run: | + julia --project=bindings/julia/encoding3d -e 'using Pkg; Pkg.instantiate()' + julia --project=bindings/julia/encoding3d bindings/julia/conformance_run.jl conformance out + - uses: ./.github/actions/conformance + with: + dumps: out/dumps + written: out/written + + # encoding3d.jl as a participant of coupling sessions run by the Python coordinator (specification §11), over ZeroMQ + # and over a folder + coupling: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: "1" + - uses: julia-actions/cache@v2 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: The Python coordinator + run: pip install -e . numpy pytest pyzmq jsonschema + - name: A Julia project with encoding3d and ZMQ + run: | + mkdir -p coupling-env + julia --project=coupling-env -e 'using Pkg; Pkg.develop(path="bindings/julia/encoding3d"); Pkg.add("ZMQ")' + - name: The coupling tests, with the Julia participant + env: + ENCODING3D_JULIA: julia --project=coupling-env + run: pytest -v bindings/python/test/test_coupling.py + - name: The coupling conformance sessions, the Julia participant + run: | + julia --project=coupling-env -e 'using Pkg; Pkg.add("JSON")' + encoding3d couple-conformance conformance/coupling --participant \ + "julia --project=$PWD/coupling-env $PWD/bindings/julia/coupling_conformance.jl" diff --git a/.gitignore b/.gitignore index b92e6f0..224b8ef 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ env/ downloads/ eggs/ parts/ +!conformance/instances/parts/ bin/ var/ sdist/ @@ -111,10 +112,14 @@ test/*.png examples/**/outputs/ /dev*.md +/.devplan/ # glTF-Validator reports *.report.json # the documentation build, and its pages generated from spec/vocabulary -docs/_build/ -docs/_generated/ +doc/_build/ +doc/_generated/ + +# Julia: environments resolved locally (the codec is a library: no committed manifest) +bindings/julia/**/Manifest*.toml diff --git a/.readthedocs.yml b/.readthedocs.yml index d27053d..a160bb4 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,4 +9,4 @@ conda: environment: conda/environment.yml sphinx: - configuration: docs/conf.py + configuration: doc/conf.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..dc9f6f1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,193 @@ +# Changelog + +## Unreleased: specification 0.4 (draft), instances, derived fields, shared parts and archives + +- **A part's root node without empty children** (every codec): a part (or a frame of no data structure) wrote the + scene node with `"children": []`, which glTF forbids (glTF-Validator: one error per operator part, so the C++, Java + and Julia conformance and the validate job failed); the key is now left out when empty. The operator parts' keys + change (their bytes do): the conformance instances, their expected values and the archives regenerated. CI: the + Windows line endings of the coupling generator's JSON (`newline="\n"`), the chosen instances' evaluation compared + within 1e-12 (MKL sums a smaller batch otherwise, 1 ulp), the conda test environment with scipy and + `conformance_run.py`, the evaluated values compared within the protocol's 1e-12 (the platform's BLAS); the + validator steps print each failing file's errors from its report. + +- **A frame written without copies** (Python): `write` streams the GLB piece by piece (`glb.write(file, document, + buffers)`: the header, the JSON chunk, then each buffer view from its array's own memory and its padding); + `glb.BufferBuilder` keeps byte views of the arrays instead of their bytes, `glb.pack` joins the same pieces once + (`to_bytes`, `part_bytes`). The files are byte-identical. The former write copied the binary chunk 4-5 times + (`tobytes` per array, the join, the padding, the chunks, the header): metafspm's use case 1 at 8 plants, day 12 + (a 39 MB frame) peaks +0.19 → +0.06 GB, with its sections +0.24 → +0.12 GB, 0.24 → 0.19 s and 0.45 → 0.37 s. + +- **A checked pack, then the folder removed** (Python): `pack(folder, path, compression=, level=, keep_folder=False)` + writes the archive next to *path*, checks it (`check_archive(path, folder)`: every file of the sequence a member and + nothing else, of the same size and CRC-32, each member read back; the archive valid, 0 errors), moves it into + place, then removes the folder; `keep_folder=True` keeps it. A failed check leaves the folder and any former + archive as they were and raises `ArchiveCheckError`; a folder holding files that are not its sequence's is refused + before writing (it would not be removed whole), and so is an archive inside the folder. `encoding3d pack ... + [--keep-folder]` (exit 1 when the check fails). The conformance run and ParaView's check keep their folders. +- **The validator on archives:** `validate_sequence` takes a sequence's `.fspm.zip` (or its folder) as `validate_file` + does: its parts are found in the archive (they were reported missing when the archive itself was given). +- **Ids across frames on an adaptive grid** (specification §8): a leaf cell is the region its `level` and `index` + give, so a cell coarsened away and refined again is the same entity and MAY have its id again; an id that comes back + MUST have the same `level` and `index`. The validator checks that instead of refusing any id that comes back on an + adaptive grid's cells (metafspm's adaptive soil, whose ids are a cell's level and corner, gave 3 errors on use case + 1's frames; none now). + +- Specification 0.4 (§12, all additions: a 0.3 file is a valid 0.4 file): **instance sets** (a data structure's + `instanceSets`: one row per instance, its template, transform, owner and own variables; vectors, one array per + instance), **linear derived fields** (f = Σ_k w_k · (A_k · v + b_k) on a template's entity set: one or several + weighted terms, offsets; operators `dense`, `csr`, or `solve`, a sparse system solved when evaluated), **shared + parts** (a template or an operator as a GLB file of its own, `FSPM_scene.part`, keyed by the SHA-256 of its bytes, + embedded in a frame, `FSPM_scene.parts`, or at `parts/.glb`, listed in the index's `parts`); §8: the folder's + layout; §8.2: a sequence packed into a `.fspm.zip` archive (ZIP64, each member compressed alone: stored, deflate, + lzma); §8.1: views' `instances` layers. Platform-neutral: any model's entities of a declared shape and linear + recovery. +- Python: `Template`, `Operator` (`dense`, `sparse`, `solve`, `of_system`), `InstanceSet`, `DerivedField`, `Ragged`, + `column_major`; `evaluate(frame, field, instances=)` (one matrix product per operator; scipy when installed, numpy + otherwise), `placed_meshes`, `Frame.part`; `write(path, frame, embed_parts=)` writing each part once and returning + the frame's keys, `write_parts`, `write_index(parts=)`, `prune_parts`; `pack`, `unpack`; `read`, `read_index`, + `read_sequence`, the validator and views reading a `.fspm.zip` like its folder (`run.fspm.zip/frame_000003.glb`); + `MissingPartError` naming the key. The validator checks every reference (parts present, valid, hashing to their + keys; templates, owners, operators' shapes, offsets). `encoding3d pack`, `unpack`, `prune`; `validate` and `info` + on instances and archives. Deflate is the default compression (measured on use case 1's frames: 11.6 % of the size + at 79 MB/s, lzma 4.1 % at 17 MB/s). +- The conformance case `instances/` (a sequence, its parts and their expected values, the evaluated fields, a frame + with its parts embedded), and `archives/`: that sequence packed stored, deflate, lzma, and with every ZIP64 record + (rewritten by `generate.py` only when their members change). `encoding3d compare-all` checks every codec's dumps of + them (frames, evaluated derived fields within a relative 1e-12, parts, each archive's frames; `--skip-instances` for + a codec reading 0.3 only, the action's `instances: "false"`), and the action validates written archives. The + conformance files say 0.4 (their expected values unchanged). +- The C++, Java and Julia codecs on 0.4: instance sets, vectors and derived fields read and written; shared parts + (templates, operators) encoded, keyed by SHA-256, embedded or under `parts/`, found through the frame (a missing + part an error naming its key); derived fields evaluated (dense, `csr`, `solve`); the index's `parts`; `.fspm.zip` + archives read (ZIP64; stored, deflate, lzma) and packed. Their conformance runs dump the instances and archives, and + write the instances again with parts they encode themselves (sequence, index, archive). Dependencies: C++ + minizip-ng (found or fetched; zlib, liblzma); Java xz-java (lzma; the rest `java.util.zip`); Julia ZipArchives.jl + and XZ_jll (liblzma's raw decoder), with SHA, SparseArrays, LinearAlgebra and Mmap. Java and Julia pack stored or + deflate (lzma read only), C++ all three. +- ParaView: instance sets drawn as their placed templates (their own variables, their owners' and the ids containing + them, their templates' constants), `instances` layers, `.fspm.zip` archives opened as folders; **encoding3d Select at + scale** rebuilds the derived fields of the instances it selects (Isolate, Show In Panel, Put Back unchanged); the + reader's **Evaluate Derived Fields** for all; `check_in_paraview.py --instances`, by hand. +- Docs: Instances, derived fields and archives (reading, packing, how a platform writes them: a reduced network's + eliminated nodes, a coarse cell's linear recovery); tutorial 1's sections 5 (instances) and 6 (an archive). +- ParaView on large frames (2026-11-09): a hidden instances layer (metafspm's sections) is an empty block, its + instances placed by **encoding3d Select at scale** for the entities it selects only (**Expand Hidden Instances** on + the reader draws them all); an instances block holds the ids, the instance and template rows and the layers' colour + variables only, one value per triangle (**All Instance Arrays** for every variable), its arrays handed to VTK + without copies; the reader and the filter share one read of a frame (`open_frame`, the last two kept) and the + derived fields rebuilt per open frame. Use case 1 (2 plants, 25 days, pvbatch peak RSS): the frame of day 16 opens + in 0.87 GB (was 5.69), day 23 in 1.92 GB (was killed at 8 GB; 1 GB of it the frame's JSON, written compactly by + metafspm since), a root segment selected with its section in 0.72–2.0 GB at any day, its axis in 1.2–4.3 GB. + `placed_meshes`' `mesh` is the mesh's index in its template (was its name, an object array), its positions placed + without a 4-component temporary. +- encoding3d Select at scale: its Scale's default is **picked**, the picked cells' own scale (`views.PICKED_SCALE`, + `views.picked_scale`), so that Isolate and Show In Panel work before any Apply; the buttons work on the Scale applied + (said in its documentation and the how-to); Show In Panel hides the selection that Apply drew in the view, and Put + Back shows it again; the filter's `` hint (was an unknown ``): Apply + leaves the reader shown. A time step at which the selection holds no instance is said once per time step, with the + plants that have some (`missing_instances`: `display_plants`, `display_period`); a derived field that cannot be + rebuilt is an error (was a warning). `grow_entities`, `instance_rows_at`. +- The plugin colours a layer's representation by an array its data holds only (`views.block_displays`' `main_array` + among the arrays present, `arrays=None` when not known): no "Could not determine array range" for a selection without + the shoot; drawn again when the arrays present change between time steps. +- Docs: `doc/python.md`'s recorder example (`frame_period="1 d"`, was 24 steps), metafspm writing 0.4 in + `implementations.md`, which also says that only metafspm reads its extras (their `absent_rows`, `none_rows`). + +## Unreleased: frame views (specification §8.1, draft for 0.3) + +- Views, an optional companion of a sequence (`views.json`, `spec/schema/views.schema.json`): named configurations + of layers, each a mesh's, segments' or grid's entities selected by `where` conditions, coloured by a variable (or a + solid colour), with a colour map, range, log scale, opacity, visibility, representation, and tubes for segments. +- `openalea.encoding3d.views`: reading, writing and checking views (the schema, and every name against a frame), the + selection, and the display options ParaView's plugin draws and saves back (`block_displays`, + `displays_from_blocks`, `layer_from_display`). +- The ParaView reader's **Time Unit** (`d` by default, `h`, `min`, `s`): the times ParaView shows, converted from the + frames' own unit; the files are unchanged. +- The ParaView plugin is one file, `paraview/plugin_encoding3d.py` (was `encoding3d_reader.py`), with a how-to, + `paraview/README.md`; no macros (`apply_encoding3d_views.py`, `save_encoding3d_views.py` and + `show_encoding3d_selection.py` removed, their work in the plugin). +- The ParaView reader: "Views" and "Configuration" (a drop-down), one block per layer holding only its entities, a + segments layer with a `tube` as tubes (one closed strip per segment); `frame_to_vtk(frame, views=, configuration=)`, + unchanged without views. The plugin draws each layer as its block when ParaView shows the reader (Apply), with its + own colour variable, colour map, range, log scale or solid colour, opacity and visibility (the representation's + block properties, editable in the MultiBlock Inspector), drawn again when the Configuration or the views file + changes; the reader's "Save Views" button (and "Save As") writes the display back into `views.json`. +- Selection by scale in ParaView: the plugin's filter "encoding3d Select at scale" (Filters → encoding3d) takes ParaView's + selection and grows the picked cells to their entity at its **Scale** (a drop-down of the frames' scales) with + everything it contains, in every layer of the reader, hidden ones included, following the picked entities through + the time steps, drawn as the layers; its buttons Isolate (the other items hidden), Show In Panel (a view of its + own) and Put Back (one step back), one selection over another. Each cell carries the ids of the entities + containing it (`.id`), each block its data structure and scales. + +## Unreleased: specification 0.3 (draft), coupling platforms + +- Lazy reading (`read(path, lazy=True)`): the file mapped in memory, its JSON parsed, each column and mesh decoded on + first access (`LazyColumns`, `LazyList`), `frame.close()` or a `with` block; used by the ParaView reader and by + replays (only the columns links use). +- The folder transport deletes each message once read, unless the session keeps them (`keepMessages`, + `--keep-messages`); every participant (Python, Julia, C++, Java) applies it, and the conformance sessions check + that the folder is left empty. + +- Specification 0.3: the frames say 0.3 (0.1 and 0.2 files stay valid); "Changes from 0.2"; the conformance files + regenerated (their expected values unchanged). +- Links between two data structures, through a mapping a participant provides (declared, sent in its updates): + `sum` shares each source's amount among its targets by weight, `mean` gives each target its sources' weighted mean; + replayed sequences provide their mappings; a replay declares every entity set and variable of its frames. +- Coupling conformance sessions (`conformance/coupling/`, written by its `generate.py`), `encoding3d couple-conformance`, + and a participant's program in each codec; partial frames (`update`) in every codec's dump. +- The C++ participant (`encoding3d/coupling.hpp`; ZeroMQ with libzmq) and the Java one (`org.openalea.encoding3d.coupling`; + ZeroMQ with JeroMQ, optional); partial frames in both codecs. +- CPlantBox's and GroIMP's logs (the adapter's tests, the plugin's headless run) replayed in coupling sessions. +- The coordinator's lookups by id vectorized, and the Python codec's enum and text columns encoded without per-row + loops: a step of 100,000 segments takes 99 ms instead of 230 ms. + +- Coupling sessions: platforms' models run together, each in its own process, through encoding3d. Partial frames + (`FSPM_scene.update`: the sender, removed entities); the messages (`spec/schema/coupling.schema.json`): a + participant's declaration (its entity sets and rights, `owns`, `refines`, `follows`; its outputs and inputs), links + (factors, units, a change of scale), turns and updates; stages; one shared structure with id ranges and removals + down the scales; two transports, ZeroMQ and a folder of messages; a recorded sequence as a participant (replay, + held or interpolated linearly). +- The reference coordinator and participant (`openalea.encoding3d.coupling`, `encoding3d couple`), unit conversion factors + (`ucum.conversion_factor`); encoding3d.jl's participant (`Participant`, `turns`, `send_update`; ZeroMQ through the + `encoding3dZMQExt` extension), tested against the coordinator over both transports. +- The Python codec refuses enum columns (labels) with missing values instead of writing an unnamed enum value. + +## Unreleased: specification 0.2 (draft) + +- Specification 0.2, additive (0.1 files stay valid), from the bindings' points (`spec/specification.md`, "Changes + from 0.1"): missing values in integer, text and boolean columns; an entity set's `scale` and ranks as base scale + positions; reading by content; position columns in their declared unit; a data structure's `labels`; the extras + key; `edge_type` `unknown`; the `sibling_order` column; parents in another complex; bearer relations' labels and names; + dates (`format`); sparse ids; the conformance files' status and the sources' precedence. +- Vocabulary 0.2: `Seed`, `mtg_index`, the new enumerations. +- The reference codec, the validator, the ParaView reader and the conformance files (a new scene, `missing_values`). +- The Julia, C++ and Java codecs on 0.2, and their adapters using it instead of their extras: MTG.jl (missing + attributes, dates, children's order, `mtg_index`), PlantGraphs (`unknown` edges, children's order), CPlantBox (a + `mixed` Organ set, segment ends in cm), GroIMP (missing fields, scales, labels); extras keys in + lower case (`virtualplantlab`, `cplantbox`, `groimp`). + +## Specification 0.1 (draft) + +- The specification (`spec/specification.md`), its JSON Schemas and the first vocabulary (41 standard names, the + labels with their Plant Ontology terms, the enumerations, a column of names per platform). +- `openalea.encoding3d`, the Python reference codec (`bindings/python/`): reading and writing frames and sequences, the + validator, display copies, the comparison with expected values, the `encoding3d` command line. +- The conformance files (`conformance/`), their expected values, and the GitHub action checking a codec against them. +- The ParaView reader plugin (`paraview/`). +- encoding3d.jl, the Julia codec (`bindings/julia/encoding3d/`): conformant reading and writing, sequences, and a + MultiScaleTreeGraph.jl package extension (`from_mtg`, `to_mtg`); its CI, `.github/workflows/julia-codec.yml`. +- The C++ codec (`bindings/cpp/`): conformant reading and writing, sequences, a CMake package; CPlantBox's adapter + (`bindings/cpp/cplantbox/`: organisms with their geometry, soil grid and mapping; static root systems from any graph + with segment ends); its CI, `.github/workflows/cpp-codec.yml`. +- The Java codec (`bindings/java/`): conformant reading and writing, sequences; GroIMP's adapter (`bindings/java/groimp/`: + scales from refinement edges, topology, edge bits as relations, managed fields; graphs back into the model's node + classes); its CI, `.github/workflows/java-codec.yml`. +- GroIMP's plugin (`bindings/java/groimp-plugin/`): the 3D view exported as an encoding3d frame from GroIMP 3.0 (File > + Export, CLI `export3d`), its geometry meshed by GroIMP's glTF plugin, each triangle with its node; tested headless in + GroIMP 3.0.0 in CI. +- encoding3d.jl: VirtualPlantLab's graphs (PlantGraphs: `from_graph`, `to_graph`) and their turtle geometry + (PlantGeomTurtle: `add_geometry!`), as package extensions. +- encoding3d.jl: MTGs from graphs with ids from 0 or sparse (numbered 1, 2, ..., ids kept in `encoding3d_id`), and parents + in another complex kept (`encoding3d_parent`). +- The documentation (`doc/`), with a notebook; the design note (`doc/design_note.md`). diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..c8366d4 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,41 @@ +cff-version: 1.2.0 +message: "If you use encoding3d, please cite it as below." +type: software +title: "encoding3d: an exchange format for functional-structural plant model scenes" +abstract: >- + encoding3d is an exchange format for whole functional-structural plant model (FSPM) scenes, across platforms and + languages (openalea.metafspm, CPlantBox, GroIMP, VirtualPlantLab), based on glTF 2.0. Each time step is one glTF + file holding the multiscale plant graphs, their state variables and parameters, their geometry and the + environment's grids, with one shared vocabulary of variables; sequences of frames, shared parts and instances, + and archives describe a whole simulation. It also runs models of several platforms together, step by step + (coupling sessions over ZeroMQ or a folder). The repository holds the specification, its JSON schemas and + vocabulary, conformance files, a Python reference codec and validator, C++, Java and Julia codecs, and a + ParaView reader. +authors: + - family-names: "Gérault" + given-names: "Tristan" + email: "tristan.gerault@uclouvain.be" + affiliation: "UCLouvain" + orcid: "https://orcid.org/0000-0000-0000-0000" # stand-in: your ORCID + - family-names: "Pradal" + given-names: "Christophe" + email: "christophe.pradal@cirad.fr" + affiliation: "CIRAD" # stand-in: check (e.g. "CIRAD, Inria") + orcid: "https://orcid.org/0000-0000-0000-0000" # stand-in + - family-names: "Bauget" + given-names: "Fabrice" + affiliation: "Affiliation" # stand-in + orcid: "https://orcid.org/0000-0000-0000-0000" # stand-in +# The concept DOI Zenodo gives the first archived release (it resolves to the latest version); the version DOIs +# are Zenodo's, one per release, and need not be listed here. +doi: "10.5281/zenodo.0000000" # stand-in: the concept DOI once Zenodo has minted it +repository-code: "https://github.com/openalea/encoding3d" +url: "https://encoding3d.readthedocs.io" +license: CECILL-C +keywords: + - functional-structural plant model + - FSPM + - glTF + - exchange format + - model coupling + - OpenAlea diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..da16f1f --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,8 @@ +# Code of conduct + +encoding3d is developed by several plant-modelling teams together. Everyone taking part (in issues, pull requests, +reviews, meetings) is expected to follow the +[Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/): +be welcoming and respectful, give and accept constructive criticism, and focus on what is best for the community. + +Unacceptable behaviour can be reported to the maintainers listed in `pyproject.toml`; reports are kept confidential. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..12b699c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,29 @@ +# Contributing + +encoding3d is a format shared by several FSPM platforms. Contributions are welcome from each of them: to the +specification, to the vocabulary, to a codec, to the tools. + +## What goes where + +- **`spec/`**: the agreement between the platforms (the specification, its JSON Schemas, the vocabulary). A change + there is proposed as a pull request, with the conformance scene showing it (`conformance/generate.py` and the + files it writes), the validator's check and the reference codec's update. See + [Changing the format](doc/governance.md). +- **The vocabulary** (`spec/vocabulary/`): a new standard name comes with its definition, canonical unit and kind, + and is agreed by at least two platforms. Each platform fills its own column of names. +- **`bindings//`**: one codec per language, each with its own build, tests and release, maintained by its + team. A codec passes the conformance check (`conformance/README.md`) in its CI. +- **Adapters** (a platform's model to and from the tables) stay in the platforms' own repositories. + +## Developing the Python package + +```bash +mamba env create -f conda/environment.yml # the package, editable, with the tests' and the docs' tools +mamba activate encoding3d +pytest # the package's tests (bindings/python/test) +pytest --nbmake doc/notebooks # the documentation's notebooks run +python -m sphinx -b html doc doc/_build -W # the documentation, warnings as errors +python conformance/generate.py # after a change to the conformance scenes +``` + +Commit with explicit messages, one subject per commit, and open the pull request on the development branch. diff --git a/README.md b/README.md index 11e0f41..437bda7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,16 @@ -# FSPM-TF - Functional Structural Plant Model Transmission Format - based on GLTF 2.0 +# encoding3d - functional-structural plant model scenes, based on glTF 2.0 + +[![Anaconda-Server Badge](https://anaconda.org/openalea3/openalea.encoding3d/badges/license.svg)](https://anaconda.org/openalea3/openalea.encoding3d) + +[![Build Status](https://github.com/openalea/encoding3d/actions/workflows/conda-build.yml/badge.svg?branch=main)](https://github.com/openalea/encoding3d/actions/workflows/conda-build.yml?query=branch%3Amain) +[![Docs](https://readthedocs.org/projects/encoding3d/badge/?version=latest)](https://encoding3d.readthedocs.io/) + +[![Anaconda-Server Badge](https://anaconda.org/openalea3/openalea.encoding3d/badges/version.svg)](https://anaconda.org/openalea3/openalea.encoding3d) +[![Anaconda-Server Badge](https://anaconda.org/openalea3/openalea.encoding3d/badges/platforms.svg)](https://anaconda.org/openalea3/openalea.encoding3d) +[![Python Version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue)](https://www.python.org/downloads/) +[![Downloads](https://anaconda.org/openalea3/openalea.encoding3d/badges/downloads.svg)](https://anaconda.org/openalea3/openalea.encoding3d) + +[![DOI](https://zenodo.org/badge/1438519156.svg)](https://zenodo.org/badge/latestdoi/1438519156) An exchange format for whole functional-structural plant model (FSPM) scenes, across platforms and languages (openalea.metafspm, CPlantBox, GroIMP, VirtualPlantLab). It holds: @@ -8,16 +20,17 @@ An exchange format for whole functional-structural plant model (FSPM) scenes, ac - their geometry; - the environment's grids. -Each time step is one glTF file, and the variables share one vocabulary. It also aims to serve as an in-memory -interface for coupling components across platforms. +Each time step is one glTF file, and the variables share one vocabulary. It also runs models of several platforms +together, step by step (coupling sessions: a coordinator and each platform's participant, over ZeroMQ or a folder). -Status: specification 0.1, draft. +Status: specification 0.4, draft (0.1, 0.2 and 0.3 files stay valid): 0.3 adds coupling platforms (§11), 0.4 +instances of shared templates, linear derived fields, shared parts and `.fspm.zip` archives (§12, §8.2). -- [The documentation](docs/index.md): for users (viewing, checking, Python) and for platform developers +- [The documentation](doc/index.md): for users (viewing, checking, Python) and for platform developers (implementing a codec, conformance, changing the format), built with Sphinx (`.readthedocs.yml`; locally: - `python -m sphinx -b html docs docs/_build`) + `python -m sphinx -b html doc doc/_build`) - [The specification](spec/specification.md) -- [The design note](docs/design_note.md): the review of existing formats and the decisions taken +- [The design note](doc/design_note.md): the review of existing formats and the decisions taken - [Implementations](implementations.md) ## The repository @@ -28,73 +41,86 @@ change there, released as a new version. Everything else implements, checks or d | Folder | Content | Discussed with the platforms | |---|---|---| | `spec/specification.md` | the specification: data model, glTF layout, sequences, conformance | **yes** | -| `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension and of a sequence's index | **yes** | +| `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension, of a sequence's index and views, and of a coupling session's messages | **yes** | | `spec/vocabulary/` | the standard names (units, kinds, definitions, and each platform's names for them), the labels with their Plant Ontology terms, the enumerations | **yes** | -| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`) | follows `spec/` | -| `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it | no | -| `paraview/` | the ParaView reader plugin, and the script checking it in a `pvpython` | no | -| `docs/design_note.md` | how the decisions were reached | no | +| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `encoding3d compare-all` and the action `.github/actions/conformance`); `coupling/`: the coupling sessions a participant runs (`encoding3d couple-conformance`) | follows `spec/` | +| `bindings//` | each language's codec and coupling participant, with its own build, tests and release; `bindings/python/`: `openalea.encoding3d`, the reference (codec with lazy reading, validator, display copies, views, the coupling's coordinator, command line), which reads `spec/` and ships it; `bindings/julia/`: encoding3d.jl, the Julia codec, with a ZeroMQ extension and the MultiScaleTreeGraph.jl and VirtualPlantLab extensions (proposals); `bindings/cpp/`: the C++ codec, with CPlantBox's adapter (a proposal); `bindings/java/`: the Java codec, with GroIMP's adapter and plugin (proposals) | no | +| `paraview/` | the ParaView plugin (`plugin_encoding3d.py`), its how-to, and the script checking it in a `pvpython` | no | +| `doc/design_note.md` | how the decisions were reached | no | | `pyproject.toml`, `conda/` | the Python package's build, at the root next to `spec/`, which it ships | no | ## The Python package ```bash -git clone https://github.com/openalea/fspm-TF.git -cd fspm-TF +git clone https://github.com/openalea/encoding3d.git +cd encoding3d mamba env create -f conda/environment.yml # numpy and jsonschema from conda-forge, then pip install -e . -mamba activate fspm_tf +mamba activate encoding3d pytest ``` ```python -from openalea import fspm_tf +from openalea import encoding3d -frame = fspm_tf.read("scene.glb") # data structures, entity sets as numpy columns, meshes -fspm_tf.write("copy.glb", frame) +frame = encoding3d.read("scene.glb") # data structures, entity sets as numpy columns, meshes +encoding3d.write("copy.glb", frame) ``` ```bash -fspm-tf validate scene.glb # or a sequence: fspm-tf validate run/scene.fspm.json -fspm-tf info scene.glb -fspm-tf display scene.glb water_potential # float32 per-vertex copies, for viewers such as ParaView +encoding3d validate scene.glb # or a sequence: encoding3d validate run/scene.fspm.json +encoding3d info scene.glb +encoding3d display scene.glb water_potential # float32 per-vertex copies, for viewers such as ParaView ``` ## Viewing in ParaView -`paraview/fspm_tf_reader.py` is a ParaView reader plugin (ParaView ≥ 5.10): +`paraview/plugin_encoding3d.py` is encoding3d's ParaView plugin (ParaView ≥ 5.11), the only file to load: the +*encoding3d reader* and the *encoding3d Select at scale* filter, no macros. `paraview/README.md` is the how-to. -1. In ParaView, open Tools → Manage Plugins → Load New…, and choose the file. Tick "Auto Load" to keep it loaded. -2. File → Open a `.glb` frame or a sequence's `scene.fspm.json`, and pick "fspm-TF reader". +1. In ParaView, open Tools → Manage Plugins..., click Load New ..., and choose the file. Tick "Auto Load" to keep it + loaded. +2. File → Open... a `.glb` frame or a sequence's `scene.fspm.json`, pick "encoding3d reader", and Apply. What it shows: - Each data structure is a block: its meshes, its segments as lines (for a Tube filter on `radius`) and its grid. +- With a `views.json` next to the frames, one block per layer of its configuration instead (a drop-down), drawn at + Apply as the layer says: its colour variable and colour map, range, opacity, visibility, tubes for roots, each + layer editable on its own in View → MultiBlock Inspector, and saved back with the reader's "Save Views" button. - Every variable of every scale is a data array named `.`, at full precision, with no display copies needed. A sequence's frames are the time steps. - Text columns such as labels become integer codes, `._code`; the names behind the codes are in the block's field data. +- Filters → encoding3d → encoding3d Select at scale grows picked cells to their plant, axis, organ ... in every layer; + its buttons isolate the selection, show it in a panel, and put it back. -The plugin finds `openalea.fspm_tf` in ParaView's Python if it is installed there, otherwise in this clone's `bindings/python/src/`. -It needs nothing beyond what ParaView ships (numpy). In `pvpython` scripts the reader is -`fspmTFreader(FileName=...)`, after `LoadPlugin("paraview/fspm_tf_reader.py", ns=globals())`. Checked with -ParaView 6.0.1: `pvpython paraview/check_in_paraview.py conformance` (also run by the tests when -`FSPM_TF_PVPYTHON` names a pvpython). +The plugin finds `openalea.encoding3d` in ParaView's Python if it is installed there, otherwise in this clone's +`bindings/python/src/`. It needs nothing beyond what ParaView ships (numpy). In `pvpython` scripts the reader is +`encoding3dreader(FileName=...)`, after `LoadPlugin("paraview/plugin_encoding3d.py", ns=globals())`. Checked with +ParaView 6.2.0: `pvpython paraview/check_in_paraview.py conformance` (also run by the tests when +`ENCODING3D_PVPYTHON` names a pvpython). ## For another platform -Two pieces of code make a platform read and write fspm-TF: +Two pieces of code make a platform read and write encoding3d: - **A codec** for its language: GLB files, property tables, sequences, the vocabulary. It depends on `spec/` only, so it lives here, in `bindings//` (`cpp/`, `java/`, `julia/`...), with its build files, its tests and its own release, maintained by the platform's team. A change to `spec/` then updates every codec in the same pull request, and the CI shows which languages follow. -- **An adapter** between the codec's tables and the platform's own model (e.g. `openalea.metafspm.io.fspm_tf` for +- **An adapter** between the codec's tables and the platform's own model (e.g. `openalea.metafspm.io.encoding3d` for metafspm's DataStructures). It depends on the platform, so it lives in the platform's repository. +**encoding3d depends on no platform.** Its requirements and its CI install none (not even metafspm). The adapters for +CPlantBox (`bindings/cpp/cplantbox/`), GroIMP (`bindings/java/groimp/`, `bindings/java/groimp-plugin/`), MTG.jl and +VirtualPlantLab (`bindings/julia/encoding3d/ext/`) are kept here only as **first proposals for those platforms' +teams**: built and tested by hand (each one's README), they move into the platforms once adopted. + Each codec runs the same conformance check in its own tests. It reads the conformance GLBs and compares what it -reads with their `*.expected.json`, and it runs `fspm-tf validate` on what it writes. The steps are in the +reads with their `*.expected.json` (0.4's instances and archives included: the four codecs read, write and evaluate +them), and it runs `encoding3d validate` on what it writes. The steps are in the [specification, §9](spec/specification.md#9-conformance). Each platform has a column of `spec/vocabulary/standard_names.csv`, listing its own names for each standard name -(`;`-separated). The writers read their mapping from it (`openalea.fspm_tf.vocabulary.platform_names("metafspm")`), so +(`;`-separated). The writers read their mapping from it (`openalea.encoding3d.vocabulary.platform_names("metafspm")`), so the vocabulary changes without changing the platforms' code. diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt new file mode 100644 index 0000000..e2f9710 --- /dev/null +++ b/bindings/cpp/CMakeLists.txt @@ -0,0 +1,140 @@ +# encoding3d, the C++ codec of encoding3d: a library (encoding3d::encoding3d), its tests, the conformance tool, and, when +# CPlantBox is given (ENCODING3D_CPLANTBOX_SOURCE_DIR and ENCODING3D_CPLANTBOX_LIBRARY), CPlantBox's adapter. +# +# cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build +# +# Licence: GPL-3.0 (LICENSE), as CPlantBox; the rest of encoding3d is CeCILL-C. +cmake_minimum_required(VERSION 3.16) +project(encoding3d VERSION 0.1.0 LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +option(ENCODING3D_BUILD_TESTS "Build the tests and the conformance tool" ${PROJECT_IS_TOP_LEVEL}) +set(ENCODING3D_CPLANTBOX_SOURCE_DIR "" CACHE PATH "CPlantBox's source folder (its src/ holds structural/), for the adapter") +set(ENCODING3D_CPLANTBOX_LIBRARY "" CACHE FILEPATH "CPlantBox's built library (libCPlantBox), for the adapter") + +# The dependencies: nlohmann_json (header-only) and minizip-ng (ZIP64 archives, deflate with zlib, lzma with liblzma), +# each found installed or fetched +find_package(nlohmann_json 3.11 QUIET) +if(NOT nlohmann_json_FOUND) + include(FetchContent) + FetchContent_Declare(nlohmann_json + URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz + URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa) + set(JSON_Install ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(nlohmann_json) +endif() + +# minizip-ng: installed as its own package (MINIZIP::minizip-ng), or with its compatibility layer (MINIZIP::minizip, +# conda-forge's "minizip"); else fetched and built static with deflate and lzma only (zlib and liblzma found, or fetched +# by it). An installed encoding3d needs an installed minizip-ng; a fetched one is for building in the tree. +set(ENCODING3D_MINIZIP_PACKAGE "") +find_package(minizip-ng 4 CONFIG QUIET) +if(TARGET MINIZIP::minizip-ng) + set(ENCODING3D_MINIZIP_PACKAGE minizip-ng) + set(ENCODING3D_MINIZIP MINIZIP::minizip-ng) +else() + find_package(minizip 4 CONFIG QUIET) + if(TARGET MINIZIP::minizip) + set(ENCODING3D_MINIZIP_PACKAGE minizip) + set(ENCODING3D_MINIZIP MINIZIP::minizip) + endif() +endif() +if(NOT ENCODING3D_MINIZIP_PACKAGE) + include(FetchContent) + FetchContent_Declare(minizip_ng + URL https://github.com/zlib-ng/minizip-ng/archive/refs/tags/4.0.10.tar.gz + URL_HASH SHA256=c362e35ee973fa7be58cc5e38a4a6c23cc8f7e652555daf4f115a9eb2d3a6be7) + foreach(option MZ_COMPAT MZ_BZIP2 MZ_ZSTD MZ_LIBCOMP MZ_PKCRYPT MZ_WZAES MZ_OPENSSL MZ_LIBBSD MZ_ICONV) + set(${option} OFF CACHE BOOL "" FORCE) + endforeach() + foreach(option MZ_ZLIB MZ_LZMA MZ_FETCH_LIBS) + set(${option} ON CACHE BOOL "" FORCE) + endforeach() + set(SKIP_INSTALL_ALL ON) + set(ENCODING3D_SHARED_LIBS ${BUILD_SHARED_LIBS}) + set(BUILD_SHARED_LIBS OFF) + FetchContent_MakeAvailable(minizip_ng) + set(BUILD_SHARED_LIBS ${ENCODING3D_SHARED_LIBS}) + set_target_properties(minizip-ng PROPERTIES POSITION_INDEPENDENT_CODE ON) + set(ENCODING3D_MINIZIP $) +endif() +if(ENCODING3D_MINIZIP_PACKAGE) + message(STATUS "encoding3d: archives with the installed minizip-ng (package ${ENCODING3D_MINIZIP_PACKAGE})") +else() + message(STATUS "encoding3d: archives with minizip-ng, fetched") +endif() + +add_library(encoding3d + src/model.cpp + src/glb.cpp + src/codec.cpp + src/sequence.cpp + src/dump.cpp + src/parts.cpp + src/archive.cpp + src/coupling.cpp) +add_library(encoding3d::encoding3d ALIAS encoding3d) +target_include_directories(encoding3d PUBLIC + $ + $) +target_link_libraries(encoding3d PUBLIC nlohmann_json::nlohmann_json) +target_link_libraries(encoding3d PRIVATE ${ENCODING3D_MINIZIP}) + +# The coupling's ZeroMQ transport, with libzmq when found (its folder transport needs nothing) +option(ENCODING3D_WITH_ZMQ "Build the coupling's ZeroMQ transport (libzmq)" ON) +if(ENCODING3D_WITH_ZMQ) + find_path(ZMQ_INCLUDE_DIR zmq.h) + find_library(ZMQ_LIBRARY NAMES zmq libzmq) + if(ZMQ_INCLUDE_DIR AND ZMQ_LIBRARY) + target_include_directories(encoding3d PRIVATE ${ZMQ_INCLUDE_DIR}) + target_link_libraries(encoding3d PRIVATE ${ZMQ_LIBRARY}) + target_compile_definitions(encoding3d PRIVATE ENCODING3D_WITH_ZMQ) + message(STATUS "encoding3d: the coupling's ZeroMQ transport, with ${ZMQ_LIBRARY}") + else() + message(STATUS "encoding3d: libzmq not found, the coupling's folder transport only") + endif() +endif() +target_compile_features(encoding3d PUBLIC cxx_std_17) +set_target_properties(encoding3d PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(MSVC) + target_compile_options(encoding3d PRIVATE /W4 /bigobj) +else() + target_compile_options(encoding3d PRIVATE -Wall -Wextra -Wpedantic) +endif() + +# Installed as a CMake package: find_package(encoding3d) then target_link_libraries(... encoding3d::encoding3d) +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) +install(TARGETS encoding3d EXPORT encoding3dTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DATADIR}/encoding3d) +install(EXPORT encoding3dTargets NAMESPACE encoding3d:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/encoding3d) +configure_package_config_file(cmake/encoding3dConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/encoding3d) +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfigVersion.cmake + COMPATIBILITY SameMinorVersion) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/encoding3d) + +if(ENCODING3D_BUILD_TESTS) + enable_testing() + set(ENCODING3D_CONFORMANCE "${CMAKE_CURRENT_SOURCE_DIR}/../../conformance") + add_executable(encoding3d_conformance_run tools/conformance_run.cpp) + target_link_libraries(encoding3d_conformance_run PRIVATE encoding3d::encoding3d) + add_executable(encoding3d_coupling_conformance tools/coupling_conformance.cpp) + target_link_libraries(encoding3d_coupling_conformance PRIVATE encoding3d::encoding3d) + add_executable(encoding3d_tests test/test_encoding3d.cpp) + target_link_libraries(encoding3d_tests PRIVATE encoding3d::encoding3d) + add_test(NAME encoding3d_tests COMMAND encoding3d_tests "${ENCODING3D_CONFORMANCE}") +endif() + +# CPlantBox's adapter: built when CPlantBox's sources and library are given +if(ENCODING3D_CPLANTBOX_SOURCE_DIR AND ENCODING3D_CPLANTBOX_LIBRARY) + add_subdirectory(cplantbox) +endif() diff --git a/bindings/cpp/LICENSE b/bindings/cpp/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/bindings/cpp/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md new file mode 100644 index 0000000..c0604ef --- /dev/null +++ b/bindings/cpp/README.md @@ -0,0 +1,33 @@ +# encoding3d (C++) + +The C++ codec of encoding3d (C++17, CMake): reading and writing frames and sequences (spec 0.4: instance sets, derived +fields evaluated, shared parts, `.fspm.zip` archives read and packed). In `cplantbox/` is +CPlantBox's adapter, which turns an organism into a graph with its geometry, soil grid and segment-to-cell mapping, +and turns a graph into a static CPlantBox root system. + +- `include/encoding3d/`: the data model (`model.hpp`) and the codec (`encoding3d.hpp`). +- `src/`: the GLB container, reading and writing, sequences, shared parts and derived fields (`parts.cpp`, SHA-256 + included), archives (`archive.cpp`), the conformance dump. +- `tools/conformance_run.cpp`: the codec's dumps of the conformance files and its written scenes, for + `.github/actions/conformance`. +- `test/`: the tests, the conformance files included. +- `cplantbox/`: the adapter (`encoding3d::cplantbox`), built when `ENCODING3D_CPLANTBOX_SOURCE_DIR` and + `ENCODING3D_CPLANTBOX_LIBRARY` are given. + +```bash +cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build +``` + +`encoding3d/coupling.hpp` is a participant of coupling sessions (its ZeroMQ transport built when libzmq is found). + +Its dependencies, each found installed or fetched by CMake: + +- nlohmann_json (header-only, MIT); +- minizip-ng 4 (zlib licence), for the archives: ZIP64, deflate through zlib, lzma (ZIP's method 14) through liblzma. + It is found as the package `minizip-ng`, or `minizip` (conda-forge's build, with its compatibility layer); else + 4.0.10 is fetched and built static with deflate and lzma only (zlib and liblzma found, or fetched by it). + +It installs as a CMake package: `find_package(encoding3d)`, then `target_link_libraries(... encoding3d::encoding3d)`; +an installed encoding3d needs an installed minizip-ng (a fetched one is for building in the tree). See [the documentation](../../doc/cpp.md). + +Licence: GPL-3.0 (`LICENSE`), as CPlantBox; the rest of encoding3d is CeCILL-C. diff --git a/bindings/cpp/cmake/encoding3dConfig.cmake.in b/bindings/cpp/cmake/encoding3dConfig.cmake.in new file mode 100644 index 0000000..d86063d --- /dev/null +++ b/bindings/cpp/cmake/encoding3dConfig.cmake.in @@ -0,0 +1,10 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(nlohmann_json 3.11) +set(ENCODING3D_MINIZIP_PACKAGE "@ENCODING3D_MINIZIP_PACKAGE@") +if(ENCODING3D_MINIZIP_PACKAGE) + find_dependency(${ENCODING3D_MINIZIP_PACKAGE} 4 CONFIG) +endif() +include("${CMAKE_CURRENT_LIST_DIR}/encoding3dTargets.cmake") +check_required_components(encoding3d) diff --git a/bindings/cpp/cplantbox/CMakeLists.txt b/bindings/cpp/cplantbox/CMakeLists.txt new file mode 100644 index 0000000..4271763 --- /dev/null +++ b/bindings/cpp/cplantbox/CMakeLists.txt @@ -0,0 +1,28 @@ +# CPlantBox's adapter (encoding3d::cplantbox), on CPlantBox's built library and its sources' headers: +# +# cmake -S bindings/cpp -B build -DENCODING3D_CPLANTBOX_SOURCE_DIR=/path/to/CPlantBox \ +# -DENCODING3D_CPLANTBOX_LIBRARY=/path/to/build/src/libCPlantBox.a +# +# Status: a first proposal for CPlantBox's team, not a part of encoding3d (whose requirements and CI do not depend on it), +# built and tested by hand (README.md here); it moves into CPlantBox once its team adopts it. +set(CPLANTBOX_SOURCE "${ENCODING3D_CPLANTBOX_SOURCE_DIR}/src") +add_library(CPlantBox::CPlantBox UNKNOWN IMPORTED) +set_target_properties(CPlantBox::CPlantBox PROPERTIES + IMPORTED_LOCATION "${ENCODING3D_CPLANTBOX_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${CPLANTBOX_SOURCE}/structural;${CPLANTBOX_SOURCE}/functional;${CPLANTBOX_SOURCE}/visualisation;${CPLANTBOX_SOURCE}/external/tinyxml2;${CPLANTBOX_SOURCE}/external/eigen") + +add_library(encoding3d_cplantbox src/cplantbox.cpp) +add_library(encoding3d::cplantbox ALIAS encoding3d_cplantbox) +target_include_directories(encoding3d_cplantbox PUBLIC $) +target_link_libraries(encoding3d_cplantbox PUBLIC encoding3d::encoding3d CPlantBox::CPlantBox) +if(MSVC) + target_compile_definitions(encoding3d_cplantbox PRIVATE _USE_MATH_DEFINES NOMINMAX) +endif() + +if(ENCODING3D_BUILD_TESTS) + add_executable(encoding3d_cplantbox_tests test/test_cplantbox.cpp) + target_link_libraries(encoding3d_cplantbox_tests PRIVATE encoding3d::cplantbox) + add_test(NAME encoding3d_cplantbox_tests + COMMAND encoding3d_cplantbox_tests "${ENCODING3D_CONFORMANCE}" "${ENCODING3D_CPLANTBOX_SOURCE_DIR}/modelparameter/structural" + "${CMAKE_CURRENT_BINARY_DIR}/cplantbox_out") +endif() diff --git a/bindings/cpp/cplantbox/README.md b/bindings/cpp/cplantbox/README.md new file mode 100644 index 0000000..dde6e68 --- /dev/null +++ b/bindings/cpp/cplantbox/README.md @@ -0,0 +1,45 @@ +# CPlantBox's adapter: a proposal for CPlantBox + +**Status: a first proposal, for CPlantBox's team to take over.** It lives here so that the C++ binding could be built +and tested before CPlantBox's team is contacted. Once they adopt it, it moves into CPlantBox, with its tests and CI. +encoding3d itself does not depend on it: the C++ codec builds without it, and encoding3d's CI does not run it. + +What it does (`encoding3d/cplantbox.hpp`, namespace `encoding3d::cplantbox`): + +- `to_data_structure`, `to_frame`: a CPlantBox organism (plant, organs, segments, their variables) as an encoding3d + graph data structure, its ids stable while the plant grows; +- `add_geometry`: the plant's geometry (CPlantBox's PlantVisualiser) as a mesh; +- `soil_grid`, `segments_in_soil`, `add_cell_values`, `add_segment_values`, `add_node_values`: the soil grid, which + cell each segment is in, and a solver's or CPlantBox's hydraulic values; +- `to_root_system`: any platform's graph back as a static CPlantBox root system. + +## Building and testing it by hand + +It needs CPlantBox's sources and its built library. The commit it was last tested with: +`17e1fab617e72fa5e2773dfee723083bdf515053`. + +```bash +# CPlantBox's library (no Python bindings, no PiafMunch) +git clone https://github.com/Plant-Root-Soil-Interactions-Modelling/CPlantBox.git cplantbox +git -C cplantbox checkout 17e1fab617e72fa5e2773dfee723083bdf515053 +cmake -S cplantbox -B cplantbox-build -DCMAKE_BUILD_TYPE=Release -DCPB_BUILD_PYTHON=OFF -DPIAFMUNCH=OFF \ + -DCPB_BUILD_SHARED=OFF -DCPB_NATIVE_CPU_TUNING=OFF +cmake --build cplantbox-build --target CPlantBox -j 4 + +# the codec and the adapter, from encoding3d's root +cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release -DENCODING3D_CPLANTBOX_SOURCE_DIR="$PWD/cplantbox" \ + -DENCODING3D_CPLANTBOX_LIBRARY="$PWD/cplantbox-build/src/libCPlantBox.a" +cmake --build build -j 4 +ctest --test-dir build --output-on-failure + +# what it wrote is valid encoding3d; its growing log replayed in a coupling session +encoding3d validate build/cplantbox/cplantbox_out/*.glb +ENCODING3D_CPLANTBOX_LOG=build/cplantbox/cplantbox_out/anagallis_log/scene.fspm.json \ + pytest bindings/python/test/test_coupling.py -k platform_log +``` + +The tests (`test/test_cplantbox.cpp`) export a simulated root system and a plant with its geometry, soil grid and +mapping, read them back, rebuild a root system from another platform's frame, and write a growing log (a frame a day +for six days). + +Licence: GPL-3.0, as CPlantBox. diff --git a/bindings/cpp/cplantbox/include/encoding3d/cplantbox.hpp b/bindings/cpp/cplantbox/include/encoding3d/cplantbox.hpp new file mode 100644 index 0000000..5c61dbb --- /dev/null +++ b/bindings/cpp/cplantbox/include/encoding3d/cplantbox.hpp @@ -0,0 +1,106 @@ +// CPlantBox's adapter: an organism (a Plant, a RootSystem, a MappedPlant) as an encoding3d graph, its leaves' and +// stems' geometry as a mesh, its soil grid and segment-to-cell mapping; and back, a graph (CPlantBox's, or another +// platform's) as a static CPlantBox root system. +// +// Status: a first proposal for CPlantBox's team to take over, not a part of encoding3d: encoding3d's requirements and CI do not depend on it; +// it is built and tested by hand (bindings/cpp/cplantbox/README.md), and moves into CPlantBox once its team adopts it. +// +// | encoding3d | CPlantBox | +// |----------------------------------|---------------------------------------------------------------------------| +// | Plant (rank 1), one row, id 0 | the organism | +// | Organ (rank 5, scale "mixed"), | its organs: a seed, a root or a stem (a whole axis), a leaf; complex the | +// | | plant, parent its parent organ (branch) | +// | SubOrgan (rank 6), id | its segments: complex its organ, parent the segment ending at its | +// | 2^32 + distal node id | proximal node (successor along an organ, branch at a lateral's base) | +// | positions (proximal_end_*, ...) | node coordinates, in cm (declared; readers convert) | +// | other variables | CPlantBox's names and units (cm, d), declared with their standard names | +// | extras["cplantbox"] | the id offsets, the organism's parameters (its XML) | +// +// CPlantBox has two scales below the plant: organs and segments. A frame with more scales (e.g. metafspm's Plant, +// Axis, SubOrgan) is read through the entity set holding the segment ends (proximal_end_*, distal_end_*) and the +// coarser entity set its complex points to, whatever their names and ranks (see to_root_system). +#pragma once + +#include +#include +#include +#include +#include + +#include "encoding3d/encoding3d.hpp" + +namespace CPlantBox { +class Organism; +class MappedPlant; +class MappedSegments; +class Plant; +} // namespace CPlantBox + +namespace encoding3d::cplantbox { + +/// The name of CPlantBox's entry in a data structure's extras, and of its column in the vocabulary. +inline constexpr const char *PLATFORM = "cplantbox"; +/// Organ ids are CPlantBox's organ ids plus this offset (the plant is 0). +inline constexpr std::int64_t ORGAN_ID_OFFSET = 1; +/// Segment ids are their distal node's id plus this offset: stable while the plant grows, distinct from organ ids. +inline constexpr std::int64_t SEGMENT_ID_OFFSET = std::int64_t(1) << 32; + +struct Options { + std::string id = "plant"; ///< the data structure's id + std::optional model; ///< e.g. the parameter file's name + bool parameters_in_extras = true; ///< the organism's parameters (XML) in extras, to rebuild it +}; + +/// The organism as a graph data structure: its plant, organs and segments, with their variables. +DataStructure to_data_structure(const CPlantBox::Organism &organism, const Options &options = {}); + +/// The organism at its simulation time (days) as a frame of one data structure. +Frame to_frame(const CPlantBox::Organism &organism, const Options &options = {}); + +/// Adds the plant's geometry (CPlantBox's PlantVisualiser: leaf blades and stems, triangles) to its data structure, as +/// a mesh whose vertices carry their segment, organ and plant rows. resolution: PlantVisualiser's geometry +/// resolution (points around a stem), leaf_resolution: points along a leaf. +void add_geometry(DataStructure &data_structure, const std::shared_ptr &plant, + int resolution = 8, int leaf_resolution = 20); + +/// The rectangular soil grid of a MappedSegments (set by setRectangularGrid), as a regular_grid data structure of +/// `cells` (cell ids: C-order index of (x, y, z), z pointing up, metres). +DataStructure soil_grid(const CPlantBox::MappedSegments &segments, const std::string &id = "soil", + const std::string &cells = "Voxel"); + +/// The segments' soil cells (MappedSegments::seg2cell) as a mapping from the plant's SubOrgan to the soil's cells, +/// weight 1. The grid must be set without cutting segments (cut = false), so that segments keep CPlantBox's numbering. +Mapping segments_in_soil(const CPlantBox::MappedSegments &segments, const DataStructure &plant, + const DataStructure &soil, const std::string &name = "roots_in_soil"); + +/// Values to write with their declaration: e.g. a soil solver's water potential per cell, or CPlantBox's hydraulic +/// results per segment or per node (PlantHydraulicModel), declared with their units and standard names. +struct Values { + std::string name; ///< the column's name + std::vector values; ///< one per cell, segment or node, in CPlantBox's order + Variable variable; ///< its unit, standard name, kind, role +}; + +/// Adds values per soil cell (in CPlantBox's cell order: x fastest, then y, then z) to the soil grid written by +/// soil_grid, as columns of its cells (reordered to encoding3d's C order of x, y, z). +void add_cell_values(DataStructure &soil, const CPlantBox::MappedSegments &segments, const std::vector &values); + +/// Adds values per segment of the MappedSegments (e.g. PlantHydraulicModel's radial fluxes) to the plant's SubOrgan +/// set, each in the row of its segment. The segments must keep CPlantBox's numbering (a grid set without cutting). +void add_segment_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values); + +/// Adds values per node of the MappedSegments (e.g. PlantHydraulicModel's xylem potentials) to the plant's SubOrgan +/// set: each segment takes its distal node's value. +void add_node_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values); + +/// A graph as a static CPlantBox root system (StaticRoot organs on a seed, as CPlantBox's StaticPlant builds from +/// RSML): its segments' ends as the organs' nodes, a lateral at its parent's node, each organ's radius the mean of its +/// segments'. The segments: the entity set holding the segment ends (proximal_end_* and distal_end_*, by standard +/// name or by CPlantBox's names); the organs: the entity set their complex points to. Units are converted from those +/// declared (m, cm, mm). Organs that are not roots (CPlantBox's organ_type, when written by CPlantBox) are skipped: +/// CPlantBox has static roots only. +std::shared_ptr to_root_system(const DataStructure &data_structure); + +} // namespace encoding3d::cplantbox diff --git a/bindings/cpp/cplantbox/src/cplantbox.cpp b/bindings/cpp/cplantbox/src/cplantbox.cpp new file mode 100644 index 0000000..53f8296 --- /dev/null +++ b/bindings/cpp/cplantbox/src/cplantbox.cpp @@ -0,0 +1,541 @@ +#include "encoding3d/cplantbox.hpp" + +#include +#include +#include +#include +#include +#include + +#include "MappedOrganism.h" +#include "Organ.h" +#include "Organism.h" +#include "Plant.h" +#include "PlantVisualiser.h" +#include "Root.h" +#include "Seed.h" +#include "organparameter.h" +#include "rootparameter.h" +#include "tinyxml2.h" + +namespace encoding3d::cplantbox { + +namespace { + +constexpr double METRES_PER_CENTIMETRE = 0.01; + +Variable declared(const std::string &unit, const std::string &kind, const std::string &description, + std::optional standard_name = std::nullopt, const std::string &role = "state") { + Variable variable; + variable.unit = unit; + variable.kind = kind; + variable.description = description; + variable.standard_name = std::move(standard_name); + variable.role = role; + return variable; +} + +/// The vocabulary's label of a CPlantBox organ type: a root or a stem is a whole axis, a leaf an organ. +std::string organ_label(int organ_type) { + switch (organ_type) { + case CPlantBox::Organism::ot_seed: return "Seed"; + case CPlantBox::Organism::ot_root: return "RootAxis"; + case CPlantBox::Organism::ot_stem: return "ShootAxis"; + case CPlantBox::Organism::ot_leaf: return "Leaf"; + default: return "Organ"; + } +} + +std::string segment_label(int organ_type) { + switch (organ_type) { + case CPlantBox::Organism::ot_root: return "RootSegment"; + case CPlantBox::Organism::ot_stem: return "StemElement"; + case CPlantBox::Organism::ot_leaf: return "LeafElement"; + default: return "SubOrgan"; + } +} + +/// The organism's organs, seeds included, parents before their laterals. +std::vector> all_organs(const CPlantBox::Organism &organism) { + std::vector> organs; + std::set seen; + std::function &)> visit = [&](const std::shared_ptr &organ) { + if (!organ || seen.count(organ.get())) return; + seen.insert(organ.get()); + organs.push_back(organ); + for (int child = 0; child < organ->getNumberOfChildren(); ++child) visit(organ->getChild(child)); + }; + for (const auto &organ : organism.getOrgans(-1, true)) { // from each organ's root: its seed, or itself + auto base = organ; + while (base->getParent()) base = base->getParent(); + visit(base); + } + return organs; +} + +/// The organism's parameters as CPlantBox's XML (as Organism::writeParameters writes them, skipping the empty entries +/// a MappedPlant can leave in its parameter map, on which writeParameters fails). +std::string parameters_xml(const CPlantBox::Organism &organism) { + tinyxml2::XMLDocument document; + tinyxml2::XMLElement *parameters = document.NewElement("plant"); + for (int organ_type = 0; organ_type <= CPlantBox::Organism::ot_leaf; ++organ_type) + for (const auto ¶meter : organism.getOrganRandomParameter(organ_type)) + if (parameter && (organ_type == CPlantBox::Organism::ot_seed || parameter->subType > 0)) + parameters->InsertEndChild(parameter->writeXML(document, false)); + document.InsertEndChild(parameters); + tinyxml2::XMLPrinter printer; + document.Print(&printer); + return printer.CStr(); +} + +} // namespace + +DataStructure to_data_structure(const CPlantBox::Organism &organism, const Options &options) { + const auto organs = all_organs(organism); + + // Plant: the organism + EntitySet plant("Plant", 1); + plant.scale = "Plant"; + plant.set("id", std::vector{0}); + + // Organ: one row per organ + EntitySet organ_set("Organ", 5); + organ_set.scale = "mixed"; // whole root and stem axes, leaves, the seed + std::vector organ_ids, complexes, parents; + std::vector labels, edge_types; + std::vector organ_types, sub_types, orders, parent_node_indices, node_counts; + std::vector ages, lengths, radii, creation_times; + std::vector alive, active; + for (const auto &organ : organs) { + const auto parent = organ->getParent(); + organ_ids.push_back(ORGAN_ID_OFFSET + organ->getId()); + complexes.push_back(0); + parents.push_back(parent ? ORGAN_ID_OFFSET + parent->getId() : NO_ID); + edge_types.push_back(parent ? "branch" : "none"); + labels.push_back(organ_label(organ->organType())); + organ_types.push_back(organ->organType()); + sub_types.push_back(static_cast(organ->getParameter("subType"))); + orders.push_back(static_cast(organ->getParameter("order"))); + parent_node_indices.push_back(organ->getParentNI()); + node_counts.push_back(organ->getNumberOfNodes()); + ages.push_back(organ->getAge()); + lengths.push_back(organ->getLength(true)); + radii.push_back(organ->getParameter("radius")); + creation_times.push_back(organ->getNumberOfNodes() > 0 ? organ->getNodeCT(0) : std::nan("")); + alive.push_back(organ->isAlive()); + active.push_back(organ->isActive()); + } + organ_set.set("id", organ_ids).set("label", Column(labels)).set("complex", complexes).set("parent", parents) + .set("edge_type", Column(edge_types)).set("organType", organ_types).set("subType", sub_types) + .set("order", orders).set("age", ages).set("length", lengths).set("radius", radii) + .set("creationTime", creation_times).set("parentNI", parent_node_indices).set("numberOfNodes", node_counts) + .set("alive", alive).set("active", active); + organ_set.declare("organType", declared("1", "descriptor", "CPlantBox's organ type: 1 seed, 2 root, 3 stem, 4 leaf", {}, "setting")) + .declare("subType", declared("1", "descriptor", "CPlantBox's sub type: the organ's parameter set", {}, "setting")) + .declare("order", declared("1", "descriptor", "branching order (0: a base organ)", "branching_order")) + .declare("age", declared("d", "intensive", "the organ's age", "age")) + .declare("length", declared("cm", "extensive", "the organ's realized length", "length")) + .declare("radius", declared("cm", "intensive", "the organ's radius", "radius", "parameter")) + .declare("creationTime", declared("d", "descriptor", "the creation time of the organ's first node")) + .declare("parentNI", declared("1", "descriptor", "the node index, in its parent organ, where it emerges")) + .declare("numberOfNodes", declared("1", "descriptor", "the number of its nodes")) + .declare("alive", declared("1", "descriptor", "whether it is alive")) + .declare("active", declared("1", "descriptor", "whether it still grows")); + + // SubOrgan: one row per segment, its id its distal node's + std::map segment_ending_at; // node id -> segment id + for (const auto &organ : organs) + for (int node = 1; node < organ->getNumberOfNodes(); ++node) + segment_ending_at[organ->getNodeId(node)] = SEGMENT_ID_OFFSET + organ->getNodeId(node); + EntitySet segment_set("SubOrgan", 6); + segment_set.scale = "SubOrgan"; + std::vector segment_ids, segment_complexes, segment_parents; + std::vector segment_labels, segment_edge_types; + std::vector segment_organ_types, segment_sub_types; + std::vector x1, y1, z1, x2, y2, z2, segment_radii, segment_lengths, segment_creation_times; + for (std::size_t index = 0; index < organs.size(); ++index) { + const auto &organ = organs[index]; + const double radius = radii[index]; + for (int node = 1; node < organ->getNumberOfNodes(); ++node) { + const auto proximal = organ->getNode(node - 1), distal = organ->getNode(node); + segment_ids.push_back(SEGMENT_ID_OFFSET + organ->getNodeId(node)); + segment_complexes.push_back(organ_ids[index]); + const auto parent = segment_ending_at.find(organ->getNodeId(node - 1)); + segment_parents.push_back(parent == segment_ending_at.end() ? NO_ID : parent->second); + segment_edge_types.push_back(parent == segment_ending_at.end() ? "none" : node == 1 ? "branch" : "successor"); + segment_labels.push_back(segment_label(organ->organType())); + segment_organ_types.push_back(organ->organType()); + segment_sub_types.push_back(sub_types[index]); + x1.push_back(proximal.x); + y1.push_back(proximal.y); + z1.push_back(proximal.z); + x2.push_back(distal.x); + y2.push_back(distal.y); + z2.push_back(distal.z); + segment_radii.push_back(radius); + segment_lengths.push_back(distal.minus(proximal).length()); + segment_creation_times.push_back(organ->getNodeCT(node)); + } + } + segment_set.set("id", segment_ids).set("label", Column(segment_labels)).set("complex", segment_complexes) + .set("parent", segment_parents).set("edge_type", Column(segment_edge_types)) + .set("organType", segment_organ_types).set("subType", segment_sub_types) + .set("x1", x1).set("y1", y1).set("z1", z1).set("x2", x2).set("y2", y2).set("z2", z2) + .set("radius", segment_radii).set("length", segment_lengths).set("creationTime", segment_creation_times); + segment_set.declare("organType", declared("1", "descriptor", "CPlantBox's organ type of its organ", {}, "setting")) + .declare("subType", declared("1", "descriptor", "CPlantBox's sub type of its organ", {}, "setting")) + .declare("x1", declared("cm", "descriptor", "proximal node, x", "proximal_end_x")) + .declare("y1", declared("cm", "descriptor", "proximal node, y", "proximal_end_y")) + .declare("z1", declared("cm", "descriptor", "proximal node, z", "proximal_end_z")) + .declare("x2", declared("cm", "descriptor", "distal node, x", "distal_end_x")) + .declare("y2", declared("cm", "descriptor", "distal node, y", "distal_end_y")) + .declare("z2", declared("cm", "descriptor", "distal node, z", "distal_end_z")) + .declare("radius", declared("cm", "intensive", "the segment's radius (its organ's)", "radius", "parameter")) + .declare("length", declared("cm", "extensive", "the segment's length", "length", "derived")) + .declare("creationTime", declared("d", "descriptor", "the creation time of its distal node")); + + DataStructure data_structure(options.id, "graph"); + data_structure.platform = "CPlantBox"; + data_structure.model = options.model; + data_structure.entity_sets = {plant, organ_set, segment_set}; + json extras = {{"organ_id_offset", ORGAN_ID_OFFSET}, {"segment_id_offset", SEGMENT_ID_OFFSET}, + {"length_unit", "cm"}, {"time_unit", "d"}}; + if (options.parameters_in_extras) extras["parameters"] = parameters_xml(organism); + data_structure.extras = {{PLATFORM, extras}}; + return data_structure; +} + +Frame to_frame(const CPlantBox::Organism &organism, const Options &options) { + Frame frame; + frame.data_structures.push_back(to_data_structure(organism, options)); + frame.time = organism.getSimTime(); + frame.time_unit = "d"; + frame.writer = {{"platform", "CPlantBox"}, {"adapter", "encoding3d::cplantbox"}}; + frame.vocabulary_version = "0.1"; + return frame; +} + +void add_geometry(DataStructure &data_structure, const std::shared_ptr &plant, int resolution, + int leaf_resolution) { + CPlantBox::PlantVisualiser visualiser(plant); + visualiser.SetGeometryResolution(resolution); + visualiser.SetLeafResolution(leaf_resolution); + visualiser.ComputeGeometry(); + const auto points = visualiser.GetGeometry(); // x, y, z per vertex, cm + const auto indices = visualiser.GetGeometryIndices(); + const auto node_ids = visualiser.GetGeometryNodeIds(); // the node each vertex belongs to + const auto &segments = data_structure.entity_set("SubOrgan"); + const auto &organs = data_structure.entity_set("Organ"); + std::map segment_row, organ_row; + const auto &segment_ids = segments.column("id").as(); + const auto &segment_complexes = segments.column("complex").as(); + for (std::size_t row = 0; row < segment_ids.size(); ++row) segment_row[segment_ids[row]] = std::int64_t(row); + const auto &organ_ids = organs.column("id").as(); + for (std::size_t row = 0; row < organ_ids.size(); ++row) organ_row[organ_ids[row]] = std::int64_t(row); + Mesh mesh; + mesh.name = "plant"; + mesh.positions.reserve(points.size()); + for (const double value : points) mesh.positions.push_back(value * METRES_PER_CENTIMETRE); + std::vector segment_rows, organ_rows, plant_rows; + for (std::size_t vertex = 0; vertex < mesh.vertex_count(); ++vertex) { + const auto node = vertex < node_ids.size() ? node_ids[vertex] : -1; + const auto found = segment_row.find(SEGMENT_ID_OFFSET + node); + if (found == segment_row.end()) { // a base node: no segment ends there + segment_rows.push_back(NO_ID); + organ_rows.push_back(NO_ID); + } else { + segment_rows.push_back(found->second); + organ_rows.push_back(organ_row.at(segment_complexes[static_cast(found->second)])); + } + plant_rows.push_back(0); + } + mesh.feature_rows = {{"SubOrgan", segment_rows}, {"Organ", organ_rows}, {"Plant", plant_rows}}; + mesh.indices = std::vector(indices.begin(), indices.end()); + data_structure.meshes.push_back(std::move(mesh)); +} + +DataStructure soil_grid(const CPlantBox::MappedSegments &segments, const std::string &id, const std::string &cells) { + const auto &minimum = segments.minBound, &maximum = segments.maxBound, &resolution = segments.resolution; + const std::vector shape = {std::int64_t(resolution.x), std::int64_t(resolution.y), + std::int64_t(resolution.z)}; + if (shape[0] <= 0 || shape[1] <= 0 || shape[2] <= 0) throw Error("no rectangular grid set (setRectangularGrid)"); + Grid grid; + grid.shape = shape; + grid.origin = {minimum.x * METRES_PER_CENTIMETRE, minimum.y * METRES_PER_CENTIMETRE, minimum.z * METRES_PER_CENTIMETRE}; + grid.spacing = {(maximum.x - minimum.x) / resolution.x * METRES_PER_CENTIMETRE, + (maximum.y - minimum.y) / resolution.y * METRES_PER_CENTIMETRE, + (maximum.z - minimum.z) / resolution.z * METRES_PER_CENTIMETRE}; + grid.periodic = {false, false, false}; + grid.z_up = true; + grid.cells = cells; + EntitySet voxels(cells); + std::vector ids(static_cast(shape[0] * shape[1] * shape[2])); + std::iota(ids.begin(), ids.end(), 0); + voxels.set("id", ids); + DataStructure soil(id, "regular_grid"); + soil.platform = "CPlantBox"; + soil.grid = grid; + soil.entity_sets.push_back(voxels); + return soil; +} + +Mapping segments_in_soil(const CPlantBox::MappedSegments &segments, const DataStructure &plant, + const DataStructure &soil, const std::string &name) { + const auto &resolution = segments.resolution; + const auto nx = std::int64_t(resolution.x), ny = std::int64_t(resolution.y), nz = std::int64_t(resolution.z); + const auto &segment_set = plant.entity_set("SubOrgan"); + const auto &ids = segment_set.column("id").as(); + const std::set known(ids.begin(), ids.end()); + std::vector sources, targets; + for (const auto &[segment, cell] : segments.seg2cell) { + if (cell < 0) continue; // above the soil + const auto id = SEGMENT_ID_OFFSET + segments.segments.at(static_cast(segment)).y; + if (!known.count(id)) + throw Error("segment " + std::to_string(segment) + " of the mapped segments is not in '" + plant.id + + "': set the grid without cutting segments"); + // CPlantBox's cell index has x fastest, then y, then z from the bottom: encoding3d's is C order of (x, y, z) + const std::int64_t i = cell % nx, j = (cell / nx) % ny, k = cell / (nx * ny); + sources.push_back(id); + targets.push_back((i * ny + j) * nz + k); + } + Mapping mapping(name, {plant.id, "SubOrgan"}, {soil.id, soil.grid ? soil.grid->cells : "Voxel"}, "containment"); + mapping.set("source", sources).set("target", targets).set("weight", std::vector(sources.size(), 1.0)); + return mapping; +} + +void add_cell_values(DataStructure &soil, const CPlantBox::MappedSegments &segments, const std::vector &values) { + if (!soil.grid) throw Error("'" + soil.id + "' has no grid: write it with soil_grid"); + const auto &resolution = segments.resolution; + const auto nx = std::size_t(resolution.x), ny = std::size_t(resolution.y), nz = std::size_t(resolution.z); + auto &cells = soil.entity_set(soil.grid->cells); + for (const auto &item : values) { + if (item.values.size() != nx * ny * nz) + throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, the grid has " + + std::to_string(nx * ny * nz) + " cells"); + std::vector column(item.values.size()); + for (std::size_t cell = 0; cell < item.values.size(); ++cell) { // CPlantBox: x fastest; encoding3d: C order + const std::size_t i = cell % nx, j = (cell / nx) % ny, k = cell / (nx * ny); + column[(i * ny + j) * nz + k] = item.values[cell]; + } + cells.set(item.name, column).declare(item.name, item.variable); + } +} + +namespace { + +/// The SubOrgan row of each segment of the MappedSegments (by its distal node's id), or throws. +std::vector segment_rows(const DataStructure &plant, const CPlantBox::MappedSegments &segments) { + const auto &set = plant.entity_set("SubOrgan"); + std::map row_of; + const auto &ids = set.column("id").as(); + for (std::size_t row = 0; row < ids.size(); ++row) row_of[ids[row]] = row; + std::vector rows; + for (std::size_t segment = 0; segment < segments.segments.size(); ++segment) { + const auto found = row_of.find(SEGMENT_ID_OFFSET + segments.segments[segment].y); + if (found == row_of.end()) + throw Error("segment " + std::to_string(segment) + " is not in '" + plant.id + + "': set the grid without cutting segments"); + rows.push_back(found->second); + } + return rows; +} + +void add_per_segment(DataStructure &plant, const std::vector &rows, const Values &item, + const std::vector &per_segment) { + auto &set = plant.entity_set("SubOrgan"); + std::vector column(set.row_count(), std::nan("")); // segments outside the mapped ones: NaN + for (std::size_t segment = 0; segment < rows.size(); ++segment) column[rows[segment]] = per_segment[segment]; + set.set(item.name, column).declare(item.name, item.variable); +} + +} // namespace + +void add_segment_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values) { + const auto rows = segment_rows(plant, segments); + for (const auto &item : values) { + if (item.values.size() != rows.size()) + throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, " + + std::to_string(rows.size()) + " segments"); + add_per_segment(plant, rows, item, item.values); + } +} + +void add_node_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values) { + const auto rows = segment_rows(plant, segments); + for (const auto &item : values) { + if (item.values.size() != segments.nodes.size()) + throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, " + + std::to_string(segments.nodes.size()) + " nodes"); + std::vector per_segment; // each segment: its distal node's value + for (const auto &segment : segments.segments) per_segment.push_back(item.values.at(std::size_t(segment.y))); + add_per_segment(plant, rows, item, per_segment); + } +} + +// ── A graph as a static root system ────────────────────────────────────── + +namespace { + +/// The factor converting a length unit to centimetres. +double to_centimetres(const std::string &unit) { + if (unit == "m") return 100.; + if (unit == "cm") return 1.; + if (unit == "mm") return 0.1; + if (unit == "um") return 1e-4; + throw Error("length unit '" + unit + "' not converted to cm (m, cm, mm, um)"); +} + +/// The column of `set` holding the standard name (or, failing that, named `fallback`); nullptr if none. +const std::pair *find_variable(const EntitySet &set, const std::string &standard_name, + const std::string &fallback) { + for (const auto &item : set.columns) { + auto declared = set.variables.find(item.first); + if (declared != set.variables.end() && declared->second.standard_name == standard_name) return &item; + } + for (const auto &item : set.columns) + if (item.first == fallback) return &item; + return nullptr; +} + +std::vector as_doubles(const Column &column) { + return std::visit([](const auto &values) { + using T = typename std::decay_t::value_type; + if constexpr (std::is_arithmetic_v && !std::is_same_v) + return std::vector(values.begin(), values.end()); + else + throw Error("a numeric column expected"); + return std::vector{}; + }, column.values); +} + +std::vector centimetres(const EntitySet &set, const std::pair &item) { + auto values = as_doubles(item.second); + auto declared = set.variables.find(item.first); + const double factor = to_centimetres(declared == set.variables.end() ? "m" : declared->second.unit); + for (auto &value : values) value *= factor; + return values; +} + +} // namespace + +std::shared_ptr to_root_system(const DataStructure &data_structure) { + // The segments: the entity set with the segment ends; the organs: the one their complex points to + const EntitySet *segments = nullptr; + const char *ends[6][2] = {{"proximal_end_x", "x1"}, {"proximal_end_y", "y1"}, {"proximal_end_z", "z1"}, + {"distal_end_x", "x2"}, {"distal_end_y", "y2"}, {"distal_end_z", "z2"}}; + for (const auto &set : data_structure.entity_sets) { + bool all = set.has_column("id") && set.has_column("complex"); + for (const auto &end : ends) all = all && find_variable(set, end[0], end[1]); + if (all && (!segments || (set.rank && segments->rank && *set.rank > *segments->rank))) segments = &set; + } + if (!segments) throw Error("'" + data_structure.id + "': no entity set with segment ends and complexes"); + std::vector> coordinates; + for (const auto &end : ends) coordinates.push_back(centimetres(*segments, *find_variable(*segments, end[0], end[1]))); + const auto &segment_ids = segments->column("id").as(); + const auto &segment_complexes = segments->column("complex").as(); + std::vector segment_parents(segment_ids.size(), NO_ID); + if (segments->has_column("parent")) segment_parents = segments->column("parent").as(); + std::vector segment_radii(segment_ids.size(), 0.05); + if (auto radius = find_variable(*segments, "radius", "radius")) segment_radii = centimetres(*segments, *radius); + + const EntitySet *organs = nullptr; + const std::set complex_ids(segment_complexes.begin(), segment_complexes.end()); + for (const auto &set : data_structure.entity_sets) { + if (&set == segments || !set.has_column("id")) continue; + const auto &ids = set.column("id").as(); + if (std::all_of(complex_ids.begin(), complex_ids.end(), + [&](std::int64_t id) { return std::find(ids.begin(), ids.end(), id) != ids.end(); })) { + organs = &set; + break; + } + } + std::map organ_types; // CPlantBox's own frames: roots only + if (organs && organs->has_column("organType")) { + const auto &ids = organs->column("id").as(); + const auto types = as_doubles(organs->column("organType")); + for (std::size_t row = 0; row < ids.size(); ++row) organ_types[ids[row]] = int(types[row]); + } + + // Each organ's segments, in order along it (from the one whose parent is outside the organ) + std::map row_of; + for (std::size_t row = 0; row < segment_ids.size(); ++row) row_of[segment_ids[row]] = row; + std::map> rows_of_organ; + std::map next_in_organ; + std::vector organ_order; // organs, in the order of their first segment + for (std::size_t row = 0; row < segment_ids.size(); ++row) { + const auto organ = segment_complexes[row]; + if (organ_types.count(organ) && organ_types[organ] != CPlantBox::Organism::ot_root) continue; + if (!rows_of_organ.count(organ)) organ_order.push_back(organ); + rows_of_organ[organ].push_back(row); + const auto parent = row_of.find(segment_parents[row]); + if (parent != row_of.end() && segment_complexes[parent->second] == organ) next_in_organ[parent->second] = row; + } + + auto plant = std::make_shared(); + plant->initializeReader(); + auto seed = std::make_shared(plant); + plant->addOrgan(seed); + std::map> organ_of; + std::map, int>> node_of_segment; // row -> organ, node + // Parents first: an organ's first segment's parent belongs to an organ already built + std::vector pending = organ_order, deferred; + std::size_t progress = 1; + while (!pending.empty() && progress) { + progress = 0; + deferred.clear(); + for (const auto organ : pending) { + const auto &rows = rows_of_organ[organ]; + std::size_t first = rows.front(); + for (const auto row : rows) { + const auto parent = row_of.find(segment_parents[row]); + if (parent == row_of.end() || segment_complexes[parent->second] != organ) { + first = row; + break; + } + } + const auto parent_segment = row_of.find(segment_parents[first]); + std::shared_ptr parent_organ = seed; + int parent_node = 0; + if (parent_segment != row_of.end()) { + const auto found = node_of_segment.find(parent_segment->second); + if (found == node_of_segment.end()) { // its parent organ is not built yet + deferred.push_back(organ); + continue; + } + parent_organ = found->second.first; + parent_node = found->second.second; + } + std::vector chain = {first}; + while (next_in_organ.count(chain.back())) chain.push_back(next_in_organ[chain.back()]); + double length = 0., radius = 0.; + for (const auto row : chain) { + length += std::sqrt(std::pow(coordinates[3][row] - coordinates[0][row], 2) + + std::pow(coordinates[4][row] - coordinates[1][row], 2) + + std::pow(coordinates[5][row] - coordinates[2][row], 2)); + radius += segment_radii[row] / double(chain.size()); + } + auto parameter = std::make_shared(0, 0., length, std::vector{}, + 1e6, radius, 0., 1e6, false); + auto root = std::make_shared(plant->getOrganIndex(), parameter, length, parent_node); + root->setOrganism(plant); + if (parent_segment != row_of.end()) + root->addNode(parent_organ->getNode(parent_node), parent_organ->getNodeId(parent_node), 0.); + else + root->addNode(CPlantBox::Vector3d(coordinates[0][first], coordinates[1][first], coordinates[2][first]), 0.); + for (const auto row : chain) { + root->addNode(CPlantBox::Vector3d(coordinates[3][row], coordinates[4][row], coordinates[5][row]), 0.); + node_of_segment[row] = {root, root->getNumberOfNodes() - 1}; + } + parent_organ->addChild(root); + organ_of[organ] = root; + ++progress; + } + pending = deferred; + } + if (!pending.empty()) throw Error("'" + data_structure.id + "': organs whose parent segments are not found"); + return plant; +} + +} // namespace encoding3d::cplantbox diff --git a/bindings/cpp/cplantbox/test/test_cplantbox.cpp b/bindings/cpp/cplantbox/test/test_cplantbox.cpp new file mode 100644 index 0000000..a999492 --- /dev/null +++ b/bindings/cpp/cplantbox/test/test_cplantbox.cpp @@ -0,0 +1,283 @@ +// CPlantBox's adapter: a simulated plant exported (organs, segments, variables, geometry, soil and mapping), written +// and read back; a static root system rebuilt from its own frame and from another platform's (the conformance root +// system, three scales). +// +// encoding3d_cplantbox_tests CONFORMANCE MODELPARAMETER OUT +#include +#include +#include +#include +#include +#include +#include + +#include "encoding3d/cplantbox.hpp" + +#include "MappedOrganism.h" +#include "Plant.h" + +namespace filesystem = std::filesystem; +using namespace encoding3d; + +namespace { + +int failures = 0, checks = 0; + +void check(bool condition, const std::string &what); + +/// Checks that two JSON values are equal, printing their first differences when they are not. +void check_same(const json &actual, const json &expected, const std::string &what) { + const auto differences = compare(actual, expected); + for (std::size_t item = 0; item < std::min(differences.size(), 3); ++item) + std::cerr << " " << differences[item] << "\n"; + check(differences.empty(), what); +} + +void check(bool condition, const std::string &what) { + ++checks; + if (!condition) { + ++failures; + std::cerr << "FAILED: " << what << "\n"; + } +} + +std::shared_ptr grown_plant(const std::string ¶meters, double days) { + auto plant = std::make_shared(1); + plant->readParameters(parameters, "plant", true, false); + plant->initialize(false); + plant->simulate(days, false); + return plant; +} + +/// The column of an entity set as doubles. +std::vector values(const EntitySet &set, const std::string &name) { + return std::visit([](const auto &column) { + using T = typename std::decay_t::value_type; + if constexpr (std::is_arithmetic_v && !std::is_same_v) return std::vector(column.begin(), column.end()); + return std::vector{}; + }, set.column(name).values); +} + +/// The topology is valid: ids unique, complexes and parents found, edge types consistent. +void check_topology(const DataStructure &data_structure, const std::string &what) { + std::set all; + std::size_t total = 0; + for (const auto &set : data_structure.entity_sets) { + const auto ids = set.ids(); + all.insert(ids.begin(), ids.end()); + total += ids.size(); + } + check(all.size() == total, what + ": ids unique across entity sets"); + for (const auto &set : data_structure.entity_sets) { + const auto ids = set.ids(); + const std::set own(ids.begin(), ids.end()); + if (set.has_column("parent")) { + const auto &parents = set.column("parent").as(); + const auto &edges = set.column("edge_type").as(); + bool good = true; + for (std::size_t row = 0; row < parents.size(); ++row) + good = good && (parents[row] == NO_ID ? edges[row] == "none" : own.count(parents[row]) && edges[row] != "none"); + check(good, what + ": parents of " + set.name + " in their set, with their edge types"); + } + if (set.has_column("complex")) { + bool good = true; + for (const auto complex : set.column("complex").as()) good = good && (complex == NO_ID || all.count(complex)); + check(good, what + ": complexes of " + set.name + " found"); + } + } +} + +void test_root_system(const filesystem::path ¶meters, const filesystem::path &out) { + const auto plant = grown_plant((parameters / "plant" / "Anagallis_femina_Leitner_2010.xml").string(), 30.); + cplantbox::Options options; + options.model = "Anagallis_femina_Leitner_2010"; + const Frame frame = cplantbox::to_frame(*plant, options); + const auto &data_structure = frame.data_structures[0]; + const auto &segments = data_structure.entity_set("SubOrgan"); + check(segments.row_count() == std::size_t(plant->getNumberOfSegments()), "one row per CPlantBox segment"); + check(data_structure.entity_set("Organ").row_count() >= plant->getOrgans().size(), "every organ (and the seed)"); + check_topology(data_structure, "root system"); + double total = 0.; + for (const auto length : values(segments, "length")) total += length; + double organs_total = 0.; + for (const auto length : plant->getParameter("length")) organs_total += length; + check(std::abs(total - organs_total) < 1e-9 * organs_total, "segment lengths sum to the organs' lengths"); + check(frame.time == 30., "the frame's time: the simulation time, days"); + + write_frame((out / "anagallis.glb").string(), frame); // written, read back the same + const auto read_back = read_frame((out / "anagallis.glb").string()); + check_same(frame_to_json(read_back), frame_to_json(frame), "the root system read back as written"); + + // Rebuilt as a static root system (CPlantBox has static roots only: the stem is left out), and exported again: + // the same root segments, their ends bit for bit + const auto rebuilt = cplantbox::to_root_system(read_back.data_structures[0]); + const auto organ_types = values(segments, "organType"); + const auto root_segments = std::count(organ_types.begin(), organ_types.end(), double(CPlantBox::Organism::ot_root)); + check(rebuilt->getNumberOfSegments() == root_segments, + "the static root system has every root segment (" + std::to_string(rebuilt->getNumberOfSegments()) + " of " + + std::to_string(root_segments) + ")"); + const auto again = cplantbox::to_data_structure(*rebuilt); + const auto &rebuilt_segments = again.entity_set("SubOrgan"); + double root_total = 0., rebuilt_total = 0.; + const auto lengths = values(segments, "length"); + for (std::size_t row = 0; row < lengths.size(); ++row) + if (organ_types[row] == CPlantBox::Organism::ot_root) root_total += lengths[row]; + for (const auto length : values(rebuilt_segments, "length")) rebuilt_total += length; + check(std::abs(rebuilt_total - root_total) < 1e-9 * root_total, "the static root system has the same root length"); + std::multiset ends, rebuilt_ends; // in cm both ways: bit for bit + for (const char *name : {"x2", "y2", "z2"}) { + const auto coordinates = values(segments, name); + for (std::size_t row = 0; row < coordinates.size(); ++row) + if (organ_types[row] == CPlantBox::Organism::ot_root) ends.insert(coordinates[row]); + for (const auto value : values(rebuilt_segments, name)) rebuilt_ends.insert(value); + } + check(ends == rebuilt_ends, "the static root system's segment ends are the same"); + check(data_structure.entity_set("Organ").scale == std::optional("mixed"), "the Organ set is mixed"); + check(segments.variables.at("x1").unit == "cm", "segment ends in CPlantBox's unit, cm"); +} + +void test_plant_with_geometry_and_soil(const filesystem::path ¶meters, const filesystem::path &out) { + auto plant = std::make_shared(1); + plant->readParameters((parameters / "plant" / "fspm2023.xml").string(), "plant", true, false); + plant->initialize(false); + plant->simulate(28., false); + auto data_structure = cplantbox::to_data_structure(*plant); + cplantbox::add_geometry(data_structure, plant); + check(data_structure.meshes.size() == 1 && data_structure.meshes[0].vertex_count() > 0, "a mesh of the shoot"); + const auto &rows = data_structure.meshes[0].feature_rows[0].second; + std::size_t with_segment = 0; + for (const auto row : rows) with_segment += row != NO_ID; + check(with_segment > rows.size() / 2, "most vertices carry their segment"); + check_topology(data_structure, "shoot and roots"); + + plant->setRectangularGrid(CPlantBox::Vector3d(-10, -10, -40), CPlantBox::Vector3d(10, 10, 0), + CPlantBox::Vector3d(4, 4, 8), false); + auto soil = cplantbox::soil_grid(*plant); + const auto mapping = cplantbox::segments_in_soil(*plant, data_structure, soil); + check(soil.entity_set("Voxel").row_count() == 4 * 4 * 8, "the soil grid's cells"); + check(mapping.row_count() > 0, "segments mapped to soil cells"); + bool inside = true; // each mapped segment's midpoint is in its cell + const auto &segments = data_structure.entity_set("SubOrgan"); + std::map row_of; + const auto &ids = segments.column("id").as(); + for (std::size_t row = 0; row < ids.size(); ++row) row_of[ids[row]] = row; + const auto x1 = values(segments, "x1"), y1 = values(segments, "y1"), z1 = values(segments, "z1"); + const auto x2 = values(segments, "x2"), y2 = values(segments, "y2"), z2 = values(segments, "z2"); + const auto &grid = *soil.grid; + const auto &sources = mapping.column("source").as(); + const auto &targets = mapping.column("target").as(); + for (std::size_t item = 0; item < sources.size(); ++item) { + const auto row = row_of.at(sources[item]); + const double middle[3] = {(x1[row] + x2[row]) / 200, (y1[row] + y2[row]) / 200, // cm -> m + (z1[row] + z2[row]) / 200}; + std::int64_t index[3]; + for (int axis = 0; axis < 3; ++axis) + index[axis] = std::int64_t(std::floor((middle[axis] - grid.origin[axis]) / grid.spacing[axis])); + inside = inside && targets[item] == (index[0] * grid.shape[1] + index[1]) * grid.shape[2] + index[2]; + } + check(inside, "each mapped segment's midpoint lies in its cell (C order of x, y, z)"); + + // A soil solver's values per cell and CPlantBox's hydraulics per node and segment, each where it belongs + std::vector soil_potential, xylem_potential, radial_flux; + for (std::size_t cell = 0; cell < 4 * 4 * 8; ++cell) soil_potential.push_back(-100. - double(cell)); + for (const auto &node : plant->nodes) xylem_potential.push_back(node.z); // the node's own z + for (std::size_t segment = 0; segment < plant->segments.size(); ++segment) radial_flux.push_back(double(segment)); + Variable potential; + potential.unit = "cm[H2O]"; + potential.standard_name = "soil_water_potential"; + cplantbox::add_cell_values(soil, *plant, {{"soil_water_potential", soil_potential, potential}}); + potential.standard_name = "water_potential"; + Variable flux; + flux.unit = "cm3/d"; + flux.standard_name = "root_water_uptake"; + cplantbox::add_node_values(data_structure, *plant, {{"xylem_potential", xylem_potential, potential}}); + cplantbox::add_segment_values(data_structure, *plant, {{"radial_flux", radial_flux, flux}}); + const auto cell_values = values(soil.entity_set("Voxel"), "soil_water_potential"); + bool cells_right = true; + for (std::size_t item = 0; item < sources.size(); ++item) { // each mapped segment's cell: its own value + std::size_t segment = 0; + while (cplantbox::SEGMENT_ID_OFFSET + plant->segments[segment].y != sources[item]) ++segment; + cells_right = cells_right && cell_values[std::size_t(targets[item])] == soil_potential[std::size_t(plant->seg2cell.at(int(segment)))]; + } + check(cells_right, "soil values reordered to the cells' C order"); + const auto xylem = values(data_structure.entity_set("SubOrgan"), "xylem_potential"); + const auto distal_z = values(data_structure.entity_set("SubOrgan"), "z2"); + bool nodes_right = true; + for (std::size_t segment = 0; segment < plant->segments.size(); ++segment) { + const auto row = row_of.at(cplantbox::SEGMENT_ID_OFFSET + plant->segments[segment].y); + nodes_right = nodes_right && xylem[row] == distal_z[row]; + } + check(nodes_right, "each segment takes its distal node's value"); + + Frame frame; + frame.time = plant->getSimTime(); + frame.data_structures = {data_structure, soil}; + frame.mappings = {mapping}; + write_frame((out / "fspm2023.glb").string(), frame); + const auto read_back = read_frame((out / "fspm2023.glb").string()); + check_same(frame_to_json(read_back), frame_to_json(frame), "the plant, soil and mapping read back"); +} + +void test_another_platform(const filesystem::path &conformance) { + // The conformance root system: Plant, Axis and SubOrgan scales (metafspm's layout), segments in metres + const auto frame = read_frame((conformance / "root_system.glb").string()); + const auto &data_structure = frame.data_structures[0]; + const auto rebuilt = cplantbox::to_root_system(data_structure); + const auto &segments = data_structure.entity_set("SubOrgan"); + check(rebuilt->getNumberOfSegments() == int(segments.row_count()), "another platform's segments, one each"); + check(int(rebuilt->getOrgans().size()) == int(data_structure.entity_set("Axis").row_count()), + "another platform's axes as CPlantBox organs"); + double total = 0., rebuilt_total = 0.; + for (const auto length : values(segments, "length")) total += length * 100.; // m -> cm + for (const auto length : rebuilt->getParameter("length")) rebuilt_total += length; + check(std::abs(total - rebuilt_total) < 1e-6 * total, "another platform's root length, in cm"); +} + +} // namespace + +/// CPlantBox's log of a growing root system: a frame a day for six days (out/anagallis_log/), the sequence a coupling +/// session replays (bindings/python/test/test_coupling.py, ENCODING3D_CPLANTBOX_LOG). +void test_growing_log(const filesystem::path ¶meters, const filesystem::path &out) { + auto plant = std::make_shared(1); + plant->readParameters((parameters / "plant" / "Anagallis_femina_Leitner_2010.xml").string(), "plant", true, false); + plant->initialize(false); + cplantbox::Options options; + options.model = "Anagallis_femina_Leitner_2010"; + options.parameters_in_extras = false; + std::vector frames; + std::size_t previous = 0; + bool growing = true; + for (int day = 0; day <= 6; ++day) { + if (day > 0) plant->simulate(1., false); + Frame frame = cplantbox::to_frame(*plant, options); + frame.step = day; + const auto segments = frame.data_structures[0].entity_set("SubOrgan").row_count(); + growing = growing && (day == 0 || segments > previous); + previous = segments; + frames.push_back(std::move(frame)); + } + check(growing, "the log: the root system grows every day"); + const auto index = read_index(write_sequence((out / "anagallis_log").string(), frames)); + check(index.frames.size() == 7, "the log: seven frames and their index"); +} + +int main(int argument_count, char **arguments) { + if (argument_count != 4) { + std::cerr << "usage: encoding3d_cplantbox_tests CONFORMANCE MODELPARAMETER OUT\n"; + return 2; + } + const filesystem::path conformance = arguments[1], parameters = arguments[2], out = arguments[3]; + filesystem::create_directories(out); + for (auto test : {std::function([&] { test_root_system(parameters, out); }), + std::function([&] { test_plant_with_geometry_and_soil(parameters, out); }), + std::function([&] { test_another_platform(conformance); }), + std::function([&] { test_growing_log(parameters, out); })}) { + try { + test(); + } catch (const std::exception &error) { + check(false, std::string("exception: ") + error.what()); + } + } + std::cout << checks - failures << " of " << checks << " checks passed\n"; + return failures == 0 ? 0 : 1; +} diff --git a/bindings/cpp/include/encoding3d/coupling.hpp b/bindings/cpp/include/encoding3d/coupling.hpp new file mode 100644 index 0000000..8962546 --- /dev/null +++ b/bindings/cpp/include/encoding3d/coupling.hpp @@ -0,0 +1,93 @@ +// A participant's side of a coupling session (specification §11): a platform declares what it works on, provides and +// requires, then answers each turn's inputs with its outputs, as partial frames. The coordinator is encoding3d's +// (`encoding3d couple`). +// +// encoding3d::coupling::Participant participant(declaration); // a JSON declaration; endpoint from the environment +// while (auto turn = participant.next_turn()) { // until the coordinator stops the session +// ... turn->frame: its entity sets, their entities, its inputs; the model from turn->time to turn->until +// participant.send_update(frame); // its outputs and topology changes +// } +// +// Transports: a folder ("files:path") always; ZeroMQ ("tcp://…", "ipc://…") when the library is built with libzmq +// (ENCODING3D_WITH_ZMQ). +#pragma once + +#include +#include +#include +#include +#include + +#include "encoding3d/model.hpp" + +namespace encoding3d::coupling { + +/// The version of the coupling protocol this library speaks. +inline constexpr const char *PROTOCOL_VERSION = "0.3"; +/// The environment variables a participant launched by the coordinator reads: the session's endpoint, its name. +inline constexpr const char *ENDPOINT_VARIABLE = "ENCODING3D_ENDPOINT"; +inline constexpr const char *PARTICIPANT_VARIABLE = "ENCODING3D_PARTICIPANT"; + +/// A message: its JSON header and, for turns and updates, a partial frame's GLB bytes. +struct Message { + json header; + std::optional> payload; +}; + +/// A participant's channel to the coordinator. +class Channel { + public: + virtual ~Channel() = default; + virtual void send(const json &header, const std::vector *payload = nullptr) = 0; + /// The next message; throws Error after `timeout` seconds without one (none: wait). + virtual Message receive(std::optional timeout) = 0; + /// As the session says (its welcome's keepMessages): keep the messages read, or delete them (the folder transport). + virtual void keep_messages(bool) {} +}; + +/// A participant's channel to `endpoint` ("files:path", or a ZeroMQ endpoint). +std::unique_ptr participant_channel(const std::string &endpoint, const std::string &participant); + +/// Whether this library speaks ZeroMQ (built with libzmq). +bool has_zeromq(); + +/// A turn: step (0: the initial state, phase "init"), the time it starts from and the time it reaches, and its frame, +/// the participant's inputs (frame.update->removed: the entities removed since its last turn). +struct Turn { + std::int64_t step = 0; + std::string phase; + double time = 0.0; + double until = 0.0; + Frame frame; +}; + +/// A participant in a session: joining it sends the declaration (and the links it proposes) and waits for the welcome. +/// The ids of the entities it creates are in [id_offset, id_offset + id_count). +class Participant { + public: + /// `declaration`: its JSON (spec/schema/coupling.schema.json); its participant name, if empty, and `endpoint`, if + /// empty, from the environment. + explicit Participant(json declaration, std::string endpoint = "", json links = json::array(), + std::optional timeout = std::nullopt); + /// The next turn, or none when the coordinator stopped the session (`stopped`: its reason). + std::optional next_turn(); + /// Sends the turn's outputs: per entity set, the ids and outputs of the rows sent (and the topology columns of the + /// entities it adds or changes); in frame.update->removed, the entities it removed. + void send_update(Frame frame); + /// Tells the coordinator this participant cannot go on (the session stops). + void fail(const std::string &message); + + std::string name; + std::int64_t id_offset = 0; + std::int64_t id_count = 0; + json session; + json couplings; + std::optional stopped; + + private: + std::unique_ptr channel_; + std::optional timeout_; + std::int64_t step_ = -1; +}; + +} // namespace encoding3d::coupling diff --git a/bindings/cpp/include/encoding3d/encoding3d.hpp b/bindings/cpp/include/encoding3d/encoding3d.hpp new file mode 100644 index 0000000..1913e0b --- /dev/null +++ b/bindings/cpp/include/encoding3d/encoding3d.hpp @@ -0,0 +1,129 @@ +// Reading and writing encoding3d files in C++: whole functional-structural plant model scenes (multiscale plant graphs, +// their variables and parameters, their geometry, the environment's grids) at successive time steps, one glTF 2.0 +// binary (GLB) file per time step. The specification is spec/specification.md in the encoding3d repository. +// +// read_frame(path) -> Frame write_frame(path, frame) +// from_bytes(data) -> Frame to_bytes(frame) -> bytes +// read_index(path), read_sequence(path) write_sequence(directory, frames) +// add_part(frame, part) -> key a shared part (Template, Operator) its instance sets refer to by key +// frame_part(frame, key) -> Part found embedded, else at parts/.glb next to the frame +// evaluate(frame, field) a derived field rebuilt for each instance (specification §12.3) +// pack_archive(folder, path) a sequence as one .fspm.zip archive; any path above may be +// "x.fspm.zip/frame_000003.glb", a file inside one (specification §8.2) +// frame_to_json(frame), part_to_json(part) the layout of the conformance files' expected values +#pragma once + +#include +#include +#include +#include + +#include "encoding3d/model.hpp" + +namespace encoding3d { + +// ── The GLB container ───────────────────────────────────────────────────── + +/// The bytes of a GLB file from its JSON document and its binary buffer (both padded to 8 bytes). +std::vector pack(const json &document, const std::vector &binary); +/// The JSON document (keys in their order) and the binary buffer of a GLB file's bytes. +std::pair> unpack(const std::vector &data); + +// ── Frames ──────────────────────────────────────────────────────────────── + +/// The GLB file of a frame; `embed_parts`: the parts its instance sets use in it (one self-contained file), else +/// referred to by key only (write_frame writes them to parts/). +std::vector to_bytes(const Frame &frame, bool embed_parts = false); +/// The frame of a GLB file's bytes; `parts_folder`: the folder its parts/ are in (the file's folder). +Frame from_bytes(const std::vector &data, const std::string &parts_folder = ""); +/// Writes a frame to the GLB file `path`, and each part its instance sets use to parts/.glb next to it, when not +/// there yet (`embed_parts`: into the frame instead); the keys of those parts, sorted. +std::vector write_frame(const std::string &path, const Frame &frame, bool embed_parts = false); +/// The frame of the GLB file `path`, in a folder or in an archive ("run.fspm.zip/frame_000003.glb"); its parts are +/// found next to it. +Frame read_frame(const std::string &path); +/// The keys of the parts the instance sets of a frame use, sorted. +std::vector frame_part_keys(const Frame &frame); + +// ── Instances, derived fields and shared parts (specification §12) ─────── + +/// A part's key: the SHA-256 of its GLB bytes, in lower-case hexadecimal. +std::string content_key(const std::vector &data); +/// The GLB file of a shared part: a template (a frame of its data structure) or an operator (a frame of no data +/// structure, its arrays in the binary chunk), FSPM_scene.part saying which. +std::vector part_to_bytes(const Part &part); +/// The part of a part's GLB bytes; `key`, when given, checked against the bytes' hash. +Part read_part(const std::vector &data, const std::string &key = ""); +/// Encodes `part` into the frame's part_bytes; its key, for an instance set's parts. +std::string add_part(Frame &frame, const Part &part); +/// The GLB bytes of the part `key` of a frame: embedded or added, else at parts/.glb under its parts_folder; +/// MissingPartError when it is nowhere. +std::vector frame_part_bytes(const Frame &frame, const std::string &key); +/// The part `key` of a frame, decoded (frame_part_bytes, then read_part). +Part frame_part(const Frame &frame, const std::string &key); +/// The derived field `field` evaluated for the instances at `rows` (all when empty) of the instance set holding it +/// (in `data_structure` and named `instance_set` when given, if several have one of that name): one vector each, on +/// the rows of the field's entity set in the instance's template, Σ_k w_k (A_k v + b_k) over its terms whose operator +/// is not NO_ID; NaN for an instance without any. Its parts are found through the frame (frame_part). +std::vector> evaluate(const Frame &frame, const std::string &field, + const std::string &data_structure = "", + const std::string &instance_set = "", + const std::vector &rows = {}); + +// ── Archives (specification §8.2) ───────────────────────────────────────── + +/// The end of an archive's name. +inline constexpr const char *ARCHIVE_SUFFIX = ".fspm.zip"; +/// The bytes of the file `path`, in a folder or in an archive ("x.fspm.zip/member"). +std::vector read_file_bytes(const std::string &path); +/// Whether the file `path` exists, in a folder or in an archive. +bool file_exists(const std::string &path); +/// Packs the sequence of `folder` (a folder, or its index) into the ZIP64 archive `path` (x.fspm.zip): its index, +/// views.json, its frames and the parts they use, unchanged, each compressed alone (`compression`: "deflate", +/// "stored" or "lzma"). Returns `path`. +std::string pack_archive(const std::string &folder, const std::string &path, const std::string &compression = "deflate"); + +// ── Sequences ───────────────────────────────────────────────────────────── + +/// The name of a sequence's index file. +inline constexpr const char *INDEX_FILE = "scene.fspm.json"; + +/// One frame of a sequence's index. +struct IndexEntry { + double time = 0.0; + std::int64_t step = 0; + std::string file; + std::string path; ///< the file's absolute path (when read) +}; + +/// A sequence's index: its frames, in time order, and the keys of the parts they use (listed as parts/.glb). +struct Index { + std::string version = SPECIFICATION_VERSION; + std::string time_unit = "d"; + std::vector frames; + std::vector parts; +}; + +/// Writes the index of a sequence. +void write_index(const std::string &path, const Index &index); +/// The index of a sequence (its file, its folder, or an archive x.fspm.zip), each frame with the path of its file. +Index read_index(const std::string &path); +/// Writes frames to `directory`, one GLB each (name_000000.glb, ...), their parts (parts/.glb), and their index; +/// the index's path. +std::string write_sequence(const std::string &directory, const std::vector &frames, + const std::string &name = "frame"); +/// The frames of the sequence whose index is `path`. +std::vector read_sequence(const std::string &path); + +// ── The conformance files' layout ───────────────────────────────────────── + +/// A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md): +/// every table's columns as lists, NaN as null, ±∞ as "Infinity" / "-Infinity", positions as their float values. +json frame_to_json(const Frame &frame); +/// A shared part as plain JSON values: a template's data structure (as frame_to_json's), an operator's arrays. +json part_to_json(const Part &part); +/// The differences between two JSON values, as `encoding3d compare` finds them: objects by key, arrays in order, +/// numbers exactly (1 and 1.0 equal), as "path: what differs" lines (none when equal). +std::vector compare(const json &actual, const json &expected, const std::string &path = "$"); + +} // namespace encoding3d diff --git a/bindings/cpp/include/encoding3d/model.hpp b/bindings/cpp/include/encoding3d/model.hpp new file mode 100644 index 0000000..3a71315 --- /dev/null +++ b/bindings/cpp/include/encoding3d/model.hpp @@ -0,0 +1,374 @@ +// The encoding3d data model: a frame's data structures, their tables (columns of equal length), grids and meshes. +// +// Frame one time step of a scene +// DataStructure one plant population's graph, or one environment model's grid +// EntitySet the entities of one scale (or a grid's cells): a table of columns +// Relation an edge table between entity sets (cell adjacency, a branch's bearer, ...) +// Table the data structure's scalars: a table of one row +// Grid a regular or adaptive grid's layout +// Mesh triangles, each vertex carrying the row of its entity in one or several entity sets +// InstanceSet instances of shared templates (specification §12): a table of one row per instance, its +// vectors (Ragged) and derived fields (DerivedField) +// Mapping source entity <-> target entity weights between two data structures +// Template, Operator the shared parts instance sets use, kept once per sequence and found by their key +// +// A column holds one value per row (numbers, booleans, text), or a fixed-size vector per row (`width` values per +// row, row after row). Missing values: NaN in float columns; elsewhere the column's `missing` mask (one flag per row, +// empty when none is missing). +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace encoding3d { + +using json = nlohmann::ordered_json; + +/// The version of the specification this library reads and writes. +inline constexpr const char *SPECIFICATION_VERSION = "0.4"; +/// The id of no entity: the complex or parent of an entity that has none, the feature row of a vertex with none. +inline constexpr std::int64_t NO_ID = -1; +/// The values of edge_type: the root of its scale, MTG '<', MTG '+'. +inline const std::vector EDGE_TYPES = {"none", "successor", "branch", "unknown"}; +/// The vocabulary's base scales: an entity set's scale is one of them, or "mixed". +inline const std::vector BASE_SCALES = {"Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", + "SubOrgan", "Layer", "Cell", "Compartment", "Connection"}; + +/// What the specification forbids, or a file that is not encoding3d. +struct Error : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +/// A shared part a frame refers to is nowhere it may be (specification §12.1): its key, and where it was looked for. +struct MissingPartError : Error { + std::string key; + std::vector looked; + MissingPartError(std::string part_key, std::vector looked_at); +}; + +/// The values of a column: one vector per element type. Booleans are std::vector; text, labels and edge types +/// std::vector. +using ColumnValues = std::variant, std::vector, std::vector, + std::vector, std::vector, std::vector, + std::vector, std::vector, std::vector, + std::vector, std::vector, std::vector>; + +/// A column: its values, `width` per row (1: one value per row; >= 2: a fixed-size vector per row), and which rows are +/// missing (`missing`, one flag per row, empty when none is; NaN in a float column). +struct Column { + ColumnValues values; + std::size_t width = 1; + std::vector missing; + + Column() : values(std::vector{}) {} + template Column(std::vector column_values, std::size_t width = 1) + : values(std::move(column_values)), width(width) {} + Column(std::initializer_list text) : values(std::vector(text.begin(), text.end())) {} + + /// The number of values (rows × width). + std::size_t size() const; + /// The number of rows. + std::size_t rows() const { return width == 0 ? 0 : size() / width; } + /// The values as a vector of T (throws Error when the column holds another type). + template const std::vector &as() const { + if (auto found = std::get_if>(&values)) return *found; + throw Error("a column of another type than the one asked for"); + } + template std::vector &as() { + if (auto found = std::get_if>(&values)) return *found; + throw Error("a column of another type than the one asked for"); + } + template bool holds() const { return std::holds_alternative>(values); } + /// Whether row `row` is missing (by the mask; a float's NaN is read as a value). + bool is_missing(std::size_t row) const { return !missing.empty() && missing[row]; } + bool has_missing() const; + /// Marks rows as missing (one flag per row); returns the column, for chaining. + Column &with_missing(std::vector rows) { + missing = std::move(rows); + return *this; + } + /// The column's type name in the conformance files' expected values: numpy's (float64, int64, bool...), or + /// "string". + std::string dtype() const; +}; + +/// The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its kind +/// (extensive, intensive, ...: how it maps between scales and grids), its role and its provenance. +struct Variable { + std::string unit = "1"; + std::optional standard_name; + std::optional kind; + std::string role = "state"; + std::optional description; + std::optional minimum; + std::optional maximum; + std::vector references; + std::vector doi; + std::optional format; ///< a text column of dates or times: "date", "date-time" or "time" + + bool operator==(const Variable &other) const; +}; + +enum class TableKind { scalars, entity_set, relation, mapping, instance_set }; + +/// Columns of equal length, in their order, and the declarations of its variables. +struct Table { + std::string name; + std::vector> columns; + std::map variables; + TableKind table_kind = TableKind::scalars; + + Table() = default; + explicit Table(std::string name, TableKind kind = TableKind::scalars) : name(std::move(name)), table_kind(kind) {} + + /// The number of rows (0 without columns). + std::size_t row_count() const; + bool has_column(const std::string &column_name) const; + const Column &column(const std::string &column_name) const; + Column &column(const std::string &column_name); + /// Sets the column `column_name`, replacing it or appending it; returns the table, for chaining. + Table &set(const std::string &column_name, Column values); + /// Declares the variable `column_name`. + Table &declare(const std::string &column_name, Variable variable); + /// The columns whose meaning the specification fixes, for this kind of table. + const std::vector &reserved() const; + bool is_reserved(const std::string &column_name) const; + /// The names of the variable columns (the columns but the reserved ones). + std::vector variable_names() const; + /// Throws Error when columns have different numbers of rows. + void check_lengths() const; +}; + +/// The entities of one scale of a graph (rank: the scale's rank, coarser first), or a grid's cells (no rank). +/// Reserved columns: id (int64, stable across frames, never reused), label (text), complex (the id of the containing +/// entity at a coarser scale, NO_ID if none), parent (the predecessor's id at the same scale, NO_ID if none), +/// edge_type ("none", "successor", "branch"); an adaptive grid's cells: level and index. +struct EntitySet : Table { + std::optional rank; + std::optional scale; ///< a base scale (BASE_SCALES), or "mixed" + + EntitySet() : Table("", TableKind::entity_set) {} + explicit EntitySet(std::string name, std::optional rank = std::nullopt) + : Table(std::move(name), TableKind::entity_set), rank(rank) {} + /// The ids of its entities: its id column, or a grid's row numbers (from 0) when it has none. + std::vector ids() const; +}; + +/// An edge table: source and target entity ids (in the entity sets source_set and target_set), an optional id per +/// edge, an optional relation label per edge, and edge variables. +struct Relation : Table { + std::string source_set; + std::string target_set; + + Relation() : Table("", TableKind::relation) {} + Relation(std::string name, std::string source_set, std::string target_set) + : Table(std::move(name), TableKind::relation), source_set(std::move(source_set)), + target_set(std::move(target_set)) {} +}; + +/// Weights between the entities of two data structures: source and target ids, weight. source and target: (data +/// structure id, entity set name). +struct Mapping : Table { + std::pair source; + std::pair target; + std::optional method; + + Mapping() : Table("", TableKind::mapping) {} + Mapping(std::string name, std::pair source, std::pair target, + std::optional method = std::nullopt) + : Table(std::move(name), TableKind::mapping), source(std::move(source)), target(std::move(target)), + method(std::move(method)) {} +}; + +/// A grid's layout. Regular: shape cells of size spacing from origin, along axes, periodic per axis, z pointing up or +/// down (z_up). Adaptive: the same for its base level, refined up to max_level, its leaf cells (an entity set with +/// level and index columns) explicit. cells: the name of the cells' entity set. +struct Grid { + std::string kind = "regular"; + std::vector shape; + std::vector origin; + std::vector spacing; + std::vector axes = {"x", "y", "z"}; + std::vector periodic; + bool z_up = true; + std::int64_t max_level = 0; + std::string cells = "Cell"; +}; + +/// Triangles: positions (3 per vertex, in the data structure's frame), three consecutive vertices per triangle, or +/// indices (3 per triangle, from 0) into them; feature_rows: per entity set, the row (from 0) of each vertex's +/// entity in that set, NO_ID for none; display: per variable, its float copy per vertex, from the entity set +/// display_sources[variable]. +struct Mesh { + std::string name; + std::vector positions; + std::vector>> feature_rows; + std::optional> indices; + std::vector>> display; + std::map display_sources; + + std::size_t vertex_count() const { return positions.size() / 3; } +}; + +/// One array of numbers per instance, of each instance's own length (specification §12.2): `values` holds them one +/// after the other, in the instances' order; `counts` each one's length. +struct Ragged { + std::vector values; + std::vector counts; + + Ragged() = default; + Ragged(std::vector ragged_values, std::vector ragged_counts) + : values(std::move(ragged_values)), counts(std::move(ragged_counts)) {} + /// The ragged array of one vector per instance. + static Ragged of(const std::vector> &vectors); + /// The number of instances. + std::size_t size() const { return counts.size(); } + /// Where each instance's values start in `values` (one more than size(): the end). + std::vector offsets() const; + /// The values of instance `row`. + std::vector at(std::size_t row) const; +}; + +/// A derived field (specification §12.3): a field on the entity set `entity_set` of the instances' templates, rebuilt +/// for each instance i from its vector `input` v as f = Σ_k w_ik (A_ik v + b_ik) over its `terms` K terms. `operators` +/// and `weights`: K per instance, row by row (instance i's term k at i * K + k); an operator is an index in the +/// instance set's parts, NO_ID for none. `offsets`: K names of vectors, or none, each term's offset b. +struct DerivedField { + std::string name; + std::string entity_set; + std::string input; + std::size_t terms = 1; + std::vector operators; + std::vector weights; + std::vector> offsets; + Variable variable; + + std::int64_t operator_of(std::size_t row, std::size_t term) const { return operators[row * terms + term]; } + double weight_of(std::size_t row, std::size_t term) const { return weights[row * terms + term]; } +}; + +/// Instances of shared templates (specification §12.2): one row per instance, its reserved columns `template` (the +/// index in `parts` of its template, INT64), `owner` (the id, in the entity set `owners`, of the entity it draws), +/// `transform` (16 numbers per row, the column-major 4×4 placing the template; identity when absent) and `id`, then +/// its own variables. `parts`: the keys of the parts (templates and operators) the instances use; `vectors` and +/// `derived`: its vectors and derived fields (their columns __count, __operator and __weight +/// are written from them, not held as columns). +struct InstanceSet : Table { + std::optional owners; + std::vector parts; + std::vector> vectors; + std::vector derived; + + InstanceSet() : Table("", TableKind::instance_set) {} + explicit InstanceSet(std::string name, std::optional owner_set = std::nullopt) + : Table(std::move(name), TableKind::instance_set), owners(std::move(owner_set)) {} + std::size_t count() const { return row_count(); } + const Ragged &vector(const std::string &vector_name) const; + const DerivedField &derived_field(const std::string &field_name) const; + /// The 4×4 placing instance `row`'s template, column-major (identity without a transform column). + std::array transform(std::size_t row) const; +}; + +/// One plant population's graph, or one environment model's grid. origin: its frame in the scene, a 4×4 matrix row +/// by row. extras: JSON values for the writing platform (e.g. what it needs to rebuild its own data structure); +/// other readers keep them without interpreting them. +struct DataStructure { + std::string id; + std::string kind = "graph"; + std::vector entity_sets; + std::vector relations; + std::optional scalars; + std::optional grid; + std::vector meshes; + std::array origin = {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + std::optional platform; + std::optional model; + json extras = json::object(); + std::vector> labels; ///< the platform's labels -> the vocabulary's + std::vector instance_sets; + + DataStructure() = default; + explicit DataStructure(std::string id, std::string kind = "graph") : id(std::move(id)), kind(std::move(kind)) {} + const EntitySet &entity_set(const std::string &name) const; + EntitySet &entity_set(const std::string &name); + const InstanceSet &instance_set(const std::string &name) const; + InstanceSet &instance_set(const std::string &name); +}; + +/// A template (a shared part, specification §12.1): the data structure its part file holds (id "template", kind +/// "table", or "graph" with ranks): the template's own entity sets (e.g. a section's cells, their ids and constant +/// variables), relations, and meshes in the template's coordinates. A template is a valid frame of its own. +struct Template { + DataStructure data_structure{"template", "table"}; + + const EntitySet &entity_set(const std::string &name) const { return data_structure.entity_set(name); } +}; + +/// A matrix in compressed sparse rows: row r's entries are indices[indptr[r] .. indptr[r + 1]) and their values. +struct SparseMatrix { + std::vector indptr; + std::vector indices; + std::vector values; +}; + +/// A linear operator of derived fields (a shared part, specification §12.1), rows × columns: `format` "dense" +/// (`values`, row by row), "csr" (`sparse`) or "solve" (S⁻¹ R, S `system`, rows × rows, R `right_hand_side`, rows × +/// columns, solved when evaluated). +struct Operator { + std::string format = "dense"; + std::size_t rows = 0; + std::size_t columns = 0; + std::vector values; + SparseMatrix sparse; + SparseMatrix system; + SparseMatrix right_hand_side; + + static Operator dense(std::size_t rows, std::size_t columns, std::vector row_by_row); + static Operator csr(std::size_t rows, std::size_t columns, SparseMatrix matrix); + static Operator solve(std::size_t rows, std::size_t columns, SparseMatrix system, SparseMatrix right_hand_side); +}; + +/// A shared part: a template or an operator. +using Part = std::variant; + +/// What makes a frame partial: a coupling session's message (specification §11). participant: its sender +/// ("coordinator" for a turn's inputs); its tables hold some rows and columns only. removed: (data structure id, +/// entity set name) -> the ids of the entities removed. +struct Update { + std::string participant; + std::map, std::vector> removed; +}; + +/// A scene at one time step (time in time_unit, step), or a parameter set. update: set for a partial frame, a coupling +/// session's message. unknown: the content of a read file this library does not interpret (other root extensions, +/// the root's extras), written back as it was. The shared parts its instance sets refer to by key (specification +/// §12.1) are found in `part_bytes` (key -> GLB bytes: those embedded in the file read, and those add_part encoded), +/// else at parts/.glb under `parts_folder` (the folder, or the x.fspm.zip archive, the frame was read from). +struct Frame { + std::vector data_structures; + std::vector mappings; + double time = 0.0; + std::string time_unit = "d"; + std::int64_t step = 0; + json writer = json::object(); + std::optional vocabulary_version; + std::optional update; + json unknown = json::object(); + std::map> part_bytes; + std::string parts_folder; + + const DataStructure &data_structure(const std::string &id) const; + DataStructure &data_structure(const std::string &id); +}; + +} // namespace encoding3d diff --git a/bindings/cpp/src/archive.cpp b/bindings/cpp/src/archive.cpp new file mode 100644 index 0000000..0d4c540 --- /dev/null +++ b/bindings/cpp/src/archive.cpp @@ -0,0 +1,176 @@ +// Archives (specification §8.2): a sequence packed into one .fspm.zip file, a ZIP archive (ZIP64 when it needs to be) +// whose members are the folder's files, each compressed alone (stored, deflate or lzma). A file inside an archive is +// named ".fspm.zip/"; read_file_bytes and file_exists read such paths as they read a folder's files. +// ZIP through minizip-ng (its ZIP64 records, deflate with zlib, lzma with liblzma). +#include "encoding3d/encoding3d.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace encoding3d { + +namespace { + +namespace filesystem = std::filesystem; + +/// (archive, member) when `path` is a file inside an archive ("x.fspm.zip/frame_000000.glb"), or an archive itself +/// (its index, "scene.fspm.json"); none otherwise. +std::optional> split_archive(const std::string &path) { + std::string normalised = path; + for (auto &character : normalised) + if (character == '\\') character = '/'; + const std::string suffix = ARCHIVE_SUFFIX; + if (normalised.size() >= suffix.size() && + normalised.compare(normalised.size() - suffix.size(), suffix.size(), suffix) == 0 && + filesystem::is_regular_file(path)) + return std::make_pair(path, std::string(INDEX_FILE)); + for (auto position = normalised.find(suffix + "/"); position != std::string::npos; + position = normalised.find(suffix + "/", position + 1)) { + const std::string archive = path.substr(0, position + suffix.size()); + if (filesystem::is_regular_file(archive)) { + std::string member; + for (std::size_t start = position + suffix.size() + 1; start <= normalised.size();) { + const auto end = std::min(normalised.find('/', start), normalised.size()); + const std::string part = normalised.substr(start, end - start); + if (part == "..") throw Error("member '" + normalised.substr(position + suffix.size() + 1) + + "': no '..' in an archive's paths"); + if (!part.empty() && part != ".") member += (member.empty() ? "" : "/") + part; + start = end + 1; + } + return std::make_pair(archive, member); + } + } + return std::nullopt; +} + +/// A minizip-ng reader open on an archive, closed when it goes. +struct Reader { + void *handle = nullptr; + explicit Reader(const std::string &archive) : handle(mz_zip_reader_create()) { + if (!handle) throw Error("cannot read the archive '" + archive + "'"); + if (mz_zip_reader_open_file(handle, archive.c_str()) != MZ_OK) { + mz_zip_reader_delete(&handle); + throw Error("'" + archive + "' is not a ZIP archive"); + } + } + ~Reader() { + mz_zip_reader_close(handle); + mz_zip_reader_delete(&handle); + } + Reader(const Reader &) = delete; + Reader &operator=(const Reader &) = delete; + bool locate(const std::string &member) { return mz_zip_reader_locate_entry(handle, member.c_str(), 0) == MZ_OK; } +}; + +std::vector read_plain_file(const std::string &path) { + std::ifstream file(path, std::ios::binary); + if (!file) throw Error("cannot read '" + path + "'"); + return {std::istreambuf_iterator(file), std::istreambuf_iterator()}; +} + +/// The keys of the parts a frame file's instance sets use (its JSON chunk only). +std::set part_keys_of(const std::string &path) { + std::set keys; + const json document = unpack(read_file_bytes(path)).first; + const auto &scene = document.at("extensions").at("FSPM_scene"); + for (const auto &data_structure : scene.value("dataStructures", json::array())) + for (const auto &instance_set : data_structure.value("instanceSets", json::array())) + for (const auto &key : instance_set.value("parts", json::array())) keys.insert(key.get()); + return keys; +} + +} // namespace + +std::vector read_file_bytes(const std::string &path) { + const auto inside = split_archive(path); + if (!inside) return read_plain_file(path); + Reader reader(inside->first); + if (!reader.locate(inside->second)) throw Error(inside->first + " holds no '" + inside->second + "'"); + if (mz_zip_reader_entry_open(reader.handle) != MZ_OK) + throw Error("cannot read '" + inside->second + "' in " + inside->first + " (its compression method?)"); + mz_zip_file *information = nullptr; + mz_zip_reader_entry_get_info(reader.handle, &information); + std::vector bytes; + if (information && information->uncompressed_size > 0) bytes.reserve(static_cast(information->uncompressed_size)); + std::vector block(1 << 20); + while (true) { + const auto read = mz_zip_reader_entry_read(reader.handle, block.data(), static_cast(block.size())); + if (read < 0) { + mz_zip_reader_entry_close(reader.handle); + throw Error("cannot read '" + inside->second + "' in " + inside->first + ": error " + std::to_string(read)); + } + if (read == 0) break; + bytes.insert(bytes.end(), block.begin(), block.begin() + read); + } + if (mz_zip_reader_entry_close(reader.handle) != MZ_OK) // its CRC-32 checked + throw Error("'" + inside->second + "' in " + inside->first + " is corrupt (its CRC-32 differs)"); + return bytes; +} + +bool file_exists(const std::string &path) { + const auto inside = split_archive(path); + if (!inside) return filesystem::exists(path); + Reader reader(inside->first); + return reader.locate(inside->second); +} + +std::string pack_archive(const std::string &folder, const std::string &path, const std::string &compression) { + std::uint16_t method; + if (compression == "deflate") method = MZ_COMPRESS_METHOD_DEFLATE; + else if (compression == "stored") method = MZ_COMPRESS_METHOD_STORE; + else if (compression == "lzma") method = MZ_COMPRESS_METHOD_LZMA; + else throw Error("compression '" + compression + "': deflate, lzma or stored"); + const std::string suffix = ARCHIVE_SUFFIX; + if (path.size() < suffix.size() || path.compare(path.size() - suffix.size(), suffix.size(), suffix) != 0) + throw Error("'" + path + "': an archive's name ends in " + suffix); + const filesystem::path index_path = filesystem::is_directory(folder) ? filesystem::path(folder) / INDEX_FILE + : filesystem::path(folder); + const auto base = filesystem::absolute(index_path).parent_path(); + const Index index = read_index(index_path.string()); + std::vector members = {INDEX_FILE}; + if (filesystem::exists(base / "views.json")) members.push_back("views.json"); + std::set keys(index.parts.begin(), index.parts.end()); + for (const auto &entry : index.frames) { + members.push_back(entry.file); + const auto used = part_keys_of(entry.path); + keys.insert(used.begin(), used.end()); + } + for (const auto &key : keys) members.push_back("parts/" + key + ".glb"); + for (const auto &member : members) + if (!filesystem::exists(base / member)) { + if (member.rfind("parts/", 0) == 0) + throw MissingPartError(member.substr(6, member.size() - 10), {(base / member).string()}); + throw Error((base / member).string() + " does not exist"); + } + const std::string temporary = path + ".part"; + void *writer = mz_zip_writer_create(); + if (!writer) throw Error("cannot write '" + path + "'"); + mz_zip_writer_set_compress_method(writer, method); + if (mz_zip_writer_open_file(writer, temporary.c_str(), 0, 0) != MZ_OK) { + mz_zip_writer_delete(&writer); + throw Error("cannot write '" + temporary + "'"); + } + std::int32_t status = MZ_OK; + for (const auto &member : members) { + status = mz_zip_writer_add_file(writer, (base / member).string().c_str(), member.c_str()); + if (status != MZ_OK) break; + } + const auto closed = mz_zip_writer_close(writer); + mz_zip_writer_delete(&writer); + if (status != MZ_OK || closed != MZ_OK) { + filesystem::remove(temporary); + throw Error("cannot write '" + path + "': error " + std::to_string(status != MZ_OK ? status : closed)); + } + filesystem::rename(temporary, path); + return path; +} + +} // namespace encoding3d diff --git a/bindings/cpp/src/bytes.hpp b/bindings/cpp/src/bytes.hpp new file mode 100644 index 0000000..c0bead8 --- /dev/null +++ b/bindings/cpp/src/bytes.hpp @@ -0,0 +1,56 @@ +// Little-endian values in byte buffers, whatever the machine's byte order. +#pragma once + +#include +#include +#include +#include +#include + +#include "encoding3d/model.hpp" + +namespace encoding3d::detail { + +inline bool machine_is_little_endian() { + const std::uint16_t probe = 1; + std::uint8_t first; + std::memcpy(&first, &probe, 1); + return first == 1; +} + +/// Appends `count` values of type T, little-endian. +template void append_values(std::vector &bytes, const T *values, std::size_t count) { + const std::size_t start = bytes.size(); + bytes.resize(start + count * sizeof(T)); + if (count == 0) return; + std::memcpy(bytes.data() + start, values, count * sizeof(T)); + if (!machine_is_little_endian() && sizeof(T) > 1) + for (std::size_t item = 0; item < count; ++item) + std::reverse(bytes.begin() + start + item * sizeof(T), bytes.begin() + start + (item + 1) * sizeof(T)); +} + +/// The `count` values of type T stored little-endian in `bytes` (of `available` bytes) from byte `offset`. +template +std::vector read_values(const std::uint8_t *bytes, std::size_t available, std::size_t count, + std::size_t offset = 0) { + if (offset + count * sizeof(T) > available) + throw Error("a buffer view holds " + std::to_string(available) + " bytes, " + + std::to_string(offset + count * sizeof(T)) + " needed"); + std::vector values(count); + if (count == 0) return values; + std::memcpy(values.data(), bytes + offset, count * sizeof(T)); + if (!machine_is_little_endian() && sizeof(T) > 1) + for (auto &value : values) { + auto raw = reinterpret_cast(&value); + std::reverse(raw, raw + sizeof(T)); + } + return values; +} + +inline std::uint32_t read_word(const std::vector &data, std::size_t offset) { + return read_values(data.data(), data.size(), 1, offset)[0]; +} + +inline std::size_t padding(std::size_t length) { return (8 - length % 8) % 8; } + +} // namespace encoding3d::detail diff --git a/bindings/cpp/src/codec.cpp b/bindings/cpp/src/codec.cpp new file mode 100644 index 0000000..ca4117c --- /dev/null +++ b/bindings/cpp/src/codec.cpp @@ -0,0 +1,1157 @@ +// Reading and writing frames as GLB files. Tables are EXT_structural_metadata property tables (one class per table), +// meshes carry EXT_mesh_features feature IDs, and the FSPM_scene root extension holds what these two cannot: data +// structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, +// standard names, kinds and roles. +#include "encoding3d/encoding3d.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bytes.hpp" + +namespace encoding3d { + +namespace { + +const std::string EXTENSION = "FSPM_scene"; +const std::string METADATA = "EXT_structural_metadata"; +const std::string FEATURES = "EXT_mesh_features"; +const std::string EDGE_TYPE_ENUM = "FSPM_edge_type"; +const std::vector ENUM_COLUMNS = {"label", "edge_type", "relation"}; +const std::vector ID_COLUMNS = {"id", "complex", "parent", "source", "target", "level", "index", + "sibling_order"}; +constexpr std::uint8_t BOOLEAN_NO_DATA = 255; // a boolean column with missing values: UINT8 0, 1, and 255 missing +constexpr int ARRAY_BUFFER = 34962, ELEMENT_ARRAY_BUFFER = 34963; +constexpr int FLOAT = 5126, UNSIGNED_INT = 5125, UNSIGNED_SHORT = 5123, UNSIGNED_BYTE = 5121; +constexpr std::int64_t MAX_FLOAT_FEATURE = std::int64_t(1) << 24; // feature rows are float vertex attributes +// The scene's frame is z up (metres); glTF viewers expect y up: the root node turns the scene for them (-90° about +// x, column-major). Readers of the tables use the scene's frame, not this node. +const std::vector Z_UP_TO_Y_UP = {1, 0, 0, 0, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1}; + +bool contains(const std::vector &names, const std::string &name) { + return std::find(names.begin(), names.end(), name) != names.end(); +} + +void check_identifier(const std::string &name, const std::string &what) { + static const std::regex identifier("^[a-zA-Z_][a-zA-Z0-9_]*$"); + if (!std::regex_match(name, identifier)) + throw Error(what + " '" + name + "' must be an identifier (letters, digits, '_'; not starting with a digit)"); +} + +template struct Component; +template <> struct Component { static constexpr const char *name = "INT8"; }; +template <> struct Component { static constexpr const char *name = "UINT8"; }; +template <> struct Component { static constexpr const char *name = "INT16"; }; +template <> struct Component { static constexpr const char *name = "UINT16"; }; +template <> struct Component { static constexpr const char *name = "INT32"; }; +template <> struct Component { static constexpr const char *name = "UINT32"; }; +template <> struct Component { static constexpr const char *name = "INT64"; }; +template <> struct Component { static constexpr const char *name = "UINT64"; }; +template <> struct Component { static constexpr const char *name = "FLOAT32"; }; +template <> struct Component { static constexpr const char *name = "FLOAT64"; }; + +template constexpr bool is_number_v = std::is_arithmetic_v && !std::is_same_v; + +// EXT_structural_metadata forbids NaN and infinities: NaN is written as noData, the most negative finite value; +inf +// and -inf as the largest finite value and the next one after the most negative, declared in the property's extras +// (FSPM_infinity). +template T no_data_float() { return std::numeric_limits::lowest(); } +template T positive_stand_in() { return std::numeric_limits::max(); } +template T negative_stand_in() { + return std::nextafter(std::numeric_limits::lowest(), T(0)); +} + +/// A value of T the column does not hold: its minimum (signed) or maximum (unsigned) first, then inwards. +template T free_integer(const std::vector &values, const std::vector &missing) { + std::set present; + for (std::size_t row = 0; row < values.size(); ++row) + if (missing.empty() || !missing[row]) present.insert(values[row]); + const T first = std::is_signed_v ? std::numeric_limits::min() : std::numeric_limits::max(); + const T second = std::is_signed_v ? std::numeric_limits::max() : std::numeric_limits::min(); + if (!present.count(first)) return first; + if (!present.count(second)) return second; + T candidate = first; + while (present.count(candidate)) candidate = std::is_signed_v ? T(candidate + 1) : T(candidate - 1); + return candidate; +} + +/// A string the column does not hold: the empty string first, then NUL characters. +std::string free_string(const std::vector &values, const std::vector &missing) { + std::set present; + for (std::size_t row = 0; row < values.size(); ++row) + if (missing.empty() || !missing[row]) present.insert(values[row]); + std::string candidate; + while (present.count(candidate)) candidate.push_back('\0'); + return candidate; +} + +json get(const json &object, const std::string &key, json fallback = nullptr) { + if (object.is_object()) { + auto found = object.find(key); + if (found != object.end()) return *found; + } + return fallback; +} + +/// Appends arrays to one binary buffer, each at an 8-byte aligned offset, and records their buffer views. +struct BufferBuilder { + std::vector data; + json views = json::array(); + + std::int64_t add_bytes(const std::vector &bytes, int target = 0) { + data.resize(data.size() + detail::padding(data.size()), 0); + json view = {{"buffer", 0}, {"byteOffset", data.size()}, {"byteLength", bytes.size()}}; + if (target != 0) view["target"] = target; + data.insert(data.end(), bytes.begin(), bytes.end()); + views.push_back(view); + return static_cast(views.size()) - 1; + } + template std::int64_t add(const std::vector &values, int target = 0) { + std::vector bytes; + detail::append_values(bytes, values.data(), values.size()); + return add_bytes(bytes, target); + } +}; + +json declaration_json(const Variable &variable) { + json declaration = {{"unit", variable.unit}, {"role", variable.role}}; + if (variable.standard_name) declaration["standardName"] = *variable.standard_name; + if (variable.kind) declaration["kind"] = *variable.kind; + if (variable.minimum) declaration["minimum"] = *variable.minimum; + if (variable.maximum) declaration["maximum"] = *variable.maximum; + if (!variable.references.empty()) declaration["references"] = variable.references; + if (!variable.doi.empty()) declaration["doi"] = variable.doi; + if (variable.format) declaration["format"] = *variable.format; + return declaration; +} + +std::pair> enum_codes(const std::string &name, const std::vector &values, + const std::string &class_id, json &schema, + std::string &value_type) { + if (name == "edge_type") { + std::vector codes; + for (const auto &value : values) { + auto found = std::find(EDGE_TYPES.begin(), EDGE_TYPES.end(), value); + if (found == EDGE_TYPES.end()) + throw Error("edge_type of '" + class_id + "': unknown value '" + value + + "' (allowed: none, successor, branch)"); + codes.push_back(static_cast(found - EDGE_TYPES.begin())); + } + value_type = "UINT8"; + return {EDGE_TYPE_ENUM, codes}; + } + std::set sorted(values.begin(), values.end()); + std::vector names(sorted.begin(), sorted.end()); + const std::string enum_name = class_id + "__" + name; + value_type = names.size() <= 256 ? "UINT8" : "UINT16"; + json enum_values = json::array(); + std::map code_of; + for (std::size_t code = 0; code < names.size(); ++code) { + enum_values.push_back({{"name", names[code]}, {"value", code}}); + code_of[names[code]] = static_cast(code); + } + schema["enums"][enum_name] = {{"valueType", value_type}, {"values", enum_values}}; + std::vector bytes; + for (const auto &value : values) { + const std::uint16_t code = code_of[value]; + if (value_type == "UINT8") { + const auto narrow = static_cast(code); + detail::append_values(bytes, &narrow, 1); + } else { + detail::append_values(bytes, &code, 1); + } + } + return {enum_name, bytes}; +} + +/// The class properties and property table values of a table's columns. +std::pair encode_table(const Table &table, const std::string &class_id, BufferBuilder &buffers, + json &schema) { + table.check_lengths(); + json properties = json::object(), stored = json::object(); + for (const auto &[name, column] : table.columns) { + check_identifier(name, "column of '" + table.name + "'"); + json property = json::object(); + auto declared = table.variables.find(name); + if (declared != table.variables.end() && declared->second.description) + property["description"] = *declared->second.description; + const bool reserved = table.is_reserved(name); + const std::string where = "column '" + name + "' of '" + table.name + "'"; + if (reserved && contains(ENUM_COLUMNS, name)) { + if (!column.holds()) throw Error(where + ": labels are text"); + std::string value_type; + auto [enum_name, codes] = enum_codes(name, column.as(), class_id, schema, value_type); + property["type"] = "ENUM"; + property["enumType"] = enum_name; + stored[name] = {{"values", buffers.add_bytes(codes)}}; + } else if (column.holds()) { + if (column.width != 1) throw Error(where + ": text vectors are not supported"); + property["type"] = "STRING"; + std::vector strings = column.as(); + if (column.has_missing()) { // missing text: noData, a string the column lacks + const std::string no_data = free_string(strings, column.missing); + property["noData"] = no_data; + for (std::size_t row = 0; row < strings.size(); ++row) + if (column.missing[row]) strings[row] = no_data; + } + std::vector text; + std::vector offsets = {0}; + for (const auto &value : strings) { + text.insert(text.end(), value.begin(), value.end()); + offsets.push_back(text.size()); + } + json item = {{"values", buffers.add_bytes(text)}}; + if (offsets.back() < (std::uint64_t(1) << 32)) { + item["stringOffsets"] = buffers.add(std::vector(offsets.begin(), offsets.end())); + item["stringOffsetType"] = "UINT32"; + } else { + item["stringOffsets"] = buffers.add(offsets); + item["stringOffsetType"] = "UINT64"; + } + stored[name] = item; + } else if (column.holds() && column.has_missing()) { // booleans with missing values: UINT8 0, 1, 255 + property["type"] = "SCALAR"; + property["componentType"] = "UINT8"; + property["noData"] = BOOLEAN_NO_DATA; + property["extras"] = {{"FSPM_boolean", true}}; + const auto &values = column.as(); + std::vector codes(values.size()); + for (std::size_t row = 0; row < values.size(); ++row) + codes[row] = column.missing[row] ? BOOLEAN_NO_DATA : std::uint8_t(values[row]); + stored[name] = {{"values", buffers.add(codes)}}; + } else if (column.holds()) { + if (column.width != 1) throw Error(where + ": boolean vectors are not supported"); + property["type"] = "BOOLEAN"; + const auto &values = column.as(); + std::vector packed((values.size() + 7) / 8, 0); + for (std::size_t row = 0; row < values.size(); ++row) + if (values[row]) packed[row / 8] |= static_cast(1u << (row % 8)); // bit order: little + stored[name] = {{"values", buffers.add_bytes(packed)}}; + } else { + std::visit([&](const auto &values) { + using T = typename std::decay_t::value_type; + if constexpr (is_number_v) { + if constexpr (std::is_integral_v && !std::is_same_v) { + if (reserved && (contains(ID_COLUMNS, name) || // ids are INT64, whatever the platform's + table.table_kind == TableKind::instance_set)) { // and template, owner + property["type"] = "SCALAR"; + property["componentType"] = "INT64"; + if (column.width >= 2) { + property["array"] = true; + property["count"] = column.width; + } + if (name == "complex" || name == "parent") property["noData"] = NO_ID; + stored[name] = {{"values", buffers.add(std::vector(values.begin(), values.end()))}}; + return; + } + } + property["type"] = "SCALAR"; + property["componentType"] = Component::name; + if (column.width != 1) { + if (column.width < 2) throw Error(where + ": vectors have at least 2 components"); + property["array"] = true; + property["count"] = column.width; + } + std::vector data = values; + if constexpr (std::is_floating_point_v) { + for (std::size_t row = 0; row < column.missing.size(); ++row) // a missing float: NaN + if (column.missing[row]) data[row] = std::numeric_limits::quiet_NaN(); + const bool infinite = std::any_of(data.begin(), data.end(), [](T value) { return std::isinf(value); }); + if (infinite) { + const T positive = positive_stand_in(), negative = negative_stand_in(); + if (std::any_of(data.begin(), data.end(), + [&](T value) { return value == positive || value == negative; })) + throw Error(where + " holds both infinities and the values standing for them"); + property["extras"] = {{"FSPM_infinity", {{"positive", static_cast(positive)}, + {"negative", static_cast(negative)}}}}; + for (auto &value : data) + if (std::isinf(value)) value = value > 0 ? positive : negative; + } + if (std::any_of(data.begin(), data.end(), [](T value) { return std::isnan(value); })) { + const T no_data = no_data_float(); + if (std::any_of(data.begin(), data.end(), [&](T value) { return value == no_data; })) + throw Error(where + " holds both NaN and the value standing for NaN"); + property["noData"] = static_cast(no_data); + for (auto &value : data) + if (std::isnan(value)) value = no_data; + } + } else if (reserved && (name == "complex" || name == "parent")) { + property["noData"] = NO_ID; + } else if (column.has_missing()) { // missing integers: noData, a value the column lacks + const T no_data = free_integer(data, column.missing); + property["noData"] = no_data; + for (std::size_t row = 0; row < data.size(); ++row) + if (column.missing[row]) data[row] = no_data; + } + stored[name] = {{"values", buffers.add(data)}}; + } + }, column.values); + } + if (name == "id" && table.table_kind == TableKind::entity_set) property["required"] = true; + properties[name] = property; + } + return {properties, stored}; +} + +json grid_json(const Grid &grid) { + json entry = {{"kind", grid.kind}, {"shape", grid.shape}, {"origin", grid.origin}, {"spacing", grid.spacing}, + {"axes", grid.axes}, {"periodic", grid.periodic}, {"zUp", grid.z_up}, {"cells", grid.cells}}; + if (grid.kind == "adaptive") entry["maxLevel"] = grid.max_level; + return entry; +} + +std::string display_attribute(const std::string &name) { + std::string attribute = "_" + name; + std::transform(attribute.begin(), attribute.end(), attribute.begin(), + [](unsigned char character) { return static_cast(std::toupper(character)); }); + return attribute; +} + +json encode_mesh(const Mesh &mesh, const DataStructure &data_structure, const json &entry, BufferBuilder &buffers, + json &accessors) { + const std::size_t vertices = mesh.vertex_count(); + if (mesh.positions.size() != vertices * 3) throw Error("mesh '" + mesh.name + "': 3 coordinates per vertex"); + auto accessor = [&](auto values, const char *type, int component_type, int target, std::size_t count, + bool bounds = false) { + json item = {{"bufferView", buffers.add(values, target)}, {"componentType", component_type}, + {"count", count}, {"type", type}}; + if (bounds && count > 0) { + std::vector minimum(3, std::numeric_limits::infinity()), maximum(3, -minimum[0]); + for (std::size_t vertex = 0; vertex < count; ++vertex) + for (int axis = 0; axis < 3; ++axis) { + minimum[axis] = std::min(minimum[axis], static_cast(values[vertex * 3 + axis])); + maximum[axis] = std::max(maximum[axis], static_cast(values[vertex * 3 + axis])); + } + item["min"] = minimum; + item["max"] = maximum; + } + accessors.push_back(item); + return static_cast(accessors.size()) - 1; + }; + std::vector positions(mesh.positions.begin(), mesh.positions.end()); + json attributes = {{"POSITION", accessor(positions, "VEC3", FLOAT, ARRAY_BUFFER, vertices, true)}}; + std::map tables; + for (const auto &set_entry : entry["entitySets"]) + tables[set_entry["name"].get()] = set_entry["propertyTable"].get(); + json feature_ids = json::array(); + for (std::size_t set_index = 0; set_index < mesh.feature_rows.size(); ++set_index) { + const auto &[set_name, rows] = mesh.feature_rows[set_index]; + if (!tables.count(set_name)) + throw Error("mesh '" + mesh.name + "': no entity set '" + set_name + "' in '" + data_structure.id + "'"); + if (rows.size() != vertices) + throw Error("mesh '" + mesh.name + "': feature rows of '" + set_name + "' are not one per vertex"); + const auto count = static_cast(data_structure.entity_set(set_name).row_count()); + if (count >= MAX_FLOAT_FEATURE) + throw Error("mesh '" + mesh.name + "': too many entities in '" + set_name + "', feature IDs stop at 2^24"); + std::vector stored; + std::set present; + bool has_null = false; + for (const auto row : rows) { + if (row < NO_ID || row >= count) + throw Error("mesh '" + mesh.name + "': feature rows of '" + set_name + "' outside its rows"); + has_null = has_null || row == NO_ID; + if (row != NO_ID) present.insert(row); + stored.push_back(static_cast(row == NO_ID ? count : row)); // null: one past the last row + } + attributes["_FEATURE_ID_" + std::to_string(set_index)] = accessor(stored, "SCALAR", FLOAT, ARRAY_BUFFER, vertices); + json feature = {{"featureCount", present.size()}, {"attribute", set_index}, + {"propertyTable", tables[set_name]}, {"label", set_name}}; + if (has_null) feature["nullFeatureId"] = count; + feature_ids.push_back(feature); + } + for (const auto &[name, values] : mesh.display) { + if (values.size() != vertices) + throw Error("mesh '" + mesh.name + "': display copy '" + name + "' is not one value per vertex"); + attributes[display_attribute(name)] = accessor(values, "SCALAR", FLOAT, ARRAY_BUFFER, vertices); + } + json primitive = {{"attributes", attributes}, {"mode", 4}}; + if (mesh.indices) { + std::vector indices(mesh.indices->begin(), mesh.indices->end()); + primitive["indices"] = accessor(indices, "SCALAR", UNSIGNED_INT, ELEMENT_ARRAY_BUFFER, indices.size()); + } + if (!feature_ids.empty()) primitive["extensions"] = {{FEATURES, {{"featureIds", feature_ids}}}}; + return {{"name", mesh.name}, {"primitives", json::array({primitive})}}; +} + +bool is_identity(const std::array &origin) { + for (int row = 0; row < 4; ++row) + for (int column = 0; column < 4; ++column) + if (origin[row * 4 + column] != (row == column ? 1.0 : 0.0)) return false; + return true; +} + +std::vector column_major(const std::array &origin) { + std::vector values(16); + for (int row = 0; row < 4; ++row) + for (int column = 0; column < 4; ++column) values[column * 4 + row] = origin[row * 4 + column]; + return values; +} + +/// The FSPM_scene entry of an instance set (specification §12.2), its tables added by `add_table`: the instance table +/// (its columns, each vector's __count, each derived field's __operator and __weight) and one table +/// per vector (its values, the instances' one after the other). +json encode_instance_set(const InstanceSet &instance_set, const DataStructure &data_structure, + const std::function &add_table) { + check_identifier(instance_set.name, "instance set name"); + const std::string where = "instance set '" + instance_set.name + "' of '" + data_structure.id + "'"; + if (!instance_set.has_column("template")) + throw Error(where + ": no 'template' column (each instance's index in its parts)"); + const std::size_t count = instance_set.count(); + Table table = instance_set; // its columns, and those of its vectors and derived fields + json entry = {{"name", instance_set.name}, {"parts", instance_set.parts}}; + if (instance_set.owners) entry["owners"] = *instance_set.owners; + json vectors = json::array(); + for (const auto &[name, ragged] : instance_set.vectors) { + check_identifier(name, "vector of " + where); + ragged.offsets(); // its counts checked against its values + if (ragged.size() != count) + throw Error(where + ": vector '" + name + "' has " + std::to_string(ragged.size()) + " instances, the set " + + std::to_string(count)); + table.set(name + "__count", Column(ragged.counts)); + Table values(name); + values.set("values", Column(ragged.values)); + vectors.push_back({{"name", name}, + {"propertyTable", add_table(values, data_structure.id + "__" + instance_set.name + "__" + name)}}); + } + json derived_fields = json::array(); + for (const auto &derived : instance_set.derived) { + check_identifier(derived.name, "derived field of " + where); + if (derived.terms == 0 || derived.operators.size() != count * derived.terms || + derived.weights.size() != count * derived.terms) + throw Error(where + ": derived field '" + derived.name + "' has not " + std::to_string(derived.terms) + + " operators and weights per instance"); + bool has_input = false; + for (const auto &item : instance_set.vectors) has_input = has_input || item.first == derived.input; + if (!has_input) + throw Error(where + ": derived field '" + derived.name + "' applies to vector '" + derived.input + + "', which the set does not have"); + if (!derived.offsets.empty() && derived.offsets.size() != derived.terms) + throw Error(where + ": derived field '" + derived.name + "' has not one offset per term"); + const std::size_t width = derived.terms == 1 ? 1 : derived.terms; + table.set(derived.name + "__operator", Column(derived.operators, width)); + table.set(derived.name + "__weight", Column(derived.weights, width)); + json item = {{"name", derived.name}, {"entitySet", derived.entity_set}, {"input", derived.input}, + {"terms", derived.terms}, {"variable", declaration_json(derived.variable)}}; + if (std::any_of(derived.offsets.begin(), derived.offsets.end(), [](const auto &offset) { return bool(offset); })) { + json offsets = json::array(); + for (const auto &offset : derived.offsets) offsets.push_back(offset ? json(*offset) : json(nullptr)); + item["offsets"] = offsets; + } + if (derived.variable.description) item["description"] = *derived.variable.description; + derived_fields.push_back(item); + } + entry["propertyTable"] = add_table(table, data_structure.id + "__" + instance_set.name); + if (!vectors.empty()) entry["vectors"] = vectors; + if (!derived_fields.empty()) entry["derivedFields"] = derived_fields; + return entry; +} + +/// Appends a buffer view to a builder (for the arrays of an operator part). +using PartEntry = std::function; + +std::vector encode_frame(const Frame &frame, bool embed_parts, const PartEntry &part_entry); + +} // namespace + +// ── Writing ────────────────────────────────────────────────────────────── + +std::vector to_bytes(const Frame &frame, bool embed_parts) { return encode_frame(frame, embed_parts, {}); } + +namespace { + +std::vector encode_frame(const Frame &frame, bool embed_parts, const PartEntry &part_entry) { + BufferBuilder buffers; + json edge_values = json::array(); + for (std::size_t value = 0; value < EDGE_TYPES.size(); ++value) + edge_values.push_back({{"name", EDGE_TYPES[value]}, {"value", value}}); + json schema = {{"id", "fspm_scene"}, {"classes", json::object()}, + {"enums", {{EDGE_TYPE_ENUM, {{"valueType", "UINT8"}, {"values", edge_values}}}}}}; + json tables = json::array(), variables = json::object(); + json document = {{"asset", {{"version", "2.0"}, {"generator", std::string("encoding3d (C++) ") + SPECIFICATION_VERSION}}}, + {"scene", 0}, {"scenes", json::array({{{"nodes", json::array({0})}}})}, + {"nodes", json::array({{{"name", "fspm_scene"}, {"matrix", Z_UP_TO_Y_UP}, + {"children", json::array()}}})}}; + json accessors = json::array(), meshes = json::array(); + + const std::function add_table = [&](const Table &table, + const std::string &class_id) { + check_identifier(class_id, "class"); + auto [properties, stored] = encode_table(table, class_id, buffers, schema); + schema["classes"][class_id] = {{"name", table.name}, {"properties", properties}}; + tables.push_back({{"name", table.name}, {"class", class_id}, {"count", table.row_count()}, + {"properties", stored}}); + json declared = json::object(); + for (const auto &name : table.variable_names()) { + auto found = table.variables.find(name); + if (found != table.variables.end()) declared[name] = declaration_json(found->second); + } + if (!declared.empty()) variables[class_id] = declared; + return static_cast(tables.size()) - 1; + }; + + json data_structures = json::array(); + for (const auto &data_structure : frame.data_structures) { + check_identifier(data_structure.id, "data structure id"); + json entry = {{"id", data_structure.id}, {"kind", data_structure.kind}}; + if (data_structure.platform) entry["platform"] = *data_structure.platform; + if (data_structure.model) entry["model"] = *data_structure.model; + entry["origin"] = column_major(data_structure.origin); // column-major, as glTF matrices + json sets = json::array(); + for (const auto &set : data_structure.entity_sets) { + json item = {{"name", set.name}, {"rank", nullptr}}; + if (set.rank) item["rank"] = *set.rank; + if (set.scale) item["scale"] = *set.scale; + item["propertyTable"] = add_table(set, data_structure.id + "__" + set.name); + sets.push_back(item); + } + entry["entitySets"] = sets; + if (!data_structure.relations.empty()) { + json relations = json::array(); + for (const auto &relation : data_structure.relations) + relations.push_back({{"name", relation.name}, {"sourceSet", relation.source_set}, + {"targetSet", relation.target_set}, + {"propertyTable", add_table(relation, data_structure.id + "__" + relation.name)}}); + entry["relations"] = relations; + } + if (data_structure.scalars) entry["scalars"] = add_table(*data_structure.scalars, data_structure.id + "__scalars"); + if (data_structure.grid) entry["grid"] = grid_json(*data_structure.grid); + if (!data_structure.labels.empty()) { + json labels = json::object(); + for (const auto &[label, standard] : data_structure.labels) labels[label] = standard; + entry["labels"] = labels; + } + if (!data_structure.extras.empty()) entry["extras"] = data_structure.extras; + json node = {{"name", data_structure.id}}; + if (!is_identity(data_structure.origin)) node["matrix"] = entry["origin"]; + document["nodes"].push_back(node); + const auto node_index = document["nodes"].size() - 1; + document["nodes"][0]["children"].push_back(node_index); + entry["node"] = node_index; + if (!data_structure.meshes.empty()) { + document["nodes"][node_index]["children"] = json::array(); + entry["meshes"] = json::array(); + for (const auto &mesh : data_structure.meshes) { + meshes.push_back(encode_mesh(mesh, data_structure, entry, buffers, accessors)); + document["nodes"].push_back({{"name", mesh.name}, {"mesh", meshes.size() - 1}}); + document["nodes"][node_index]["children"].push_back(document["nodes"].size() - 1); + json mesh_entry = {{"name", mesh.name}, {"mesh", meshes.size() - 1}}; + if (!mesh.display.empty()) { + json display = json::array(); + for (const auto &item : mesh.display) { + auto source = mesh.display_sources.find(item.first); + display.push_back({{"attribute", display_attribute(item.first)}, {"variable", item.first}, + {"entitySet", source == mesh.display_sources.end() ? json(nullptr) + : json(source->second)}}); + } + mesh_entry["display"] = display; + } + entry["meshes"].push_back(mesh_entry); + } + } + if (!data_structure.instance_sets.empty()) { + json instance_sets = json::array(); + for (const auto &instance_set : data_structure.instance_sets) + instance_sets.push_back(encode_instance_set(instance_set, data_structure, add_table)); + entry["instanceSets"] = instance_sets; + } + data_structures.push_back(entry); + } + + json mappings = json::array(); + for (const auto &mapping : frame.mappings) { + check_identifier(mapping.name, "mapping name"); + json entry = {{"name", mapping.name}, + {"source", {{"dataStructure", mapping.source.first}, {"entitySet", mapping.source.second}}}, + {"target", {{"dataStructure", mapping.target.first}, {"entitySet", mapping.target.second}}}}; + if (mapping.method) entry["method"] = *mapping.method; + entry["propertyTable"] = add_table(mapping, "mapping__" + mapping.name); + mappings.push_back(entry); + } + + json update = nullptr; + if (frame.update) { + check_identifier(frame.update->participant, "participant"); + update = {{"participant", frame.update->participant}}; + if (!frame.update->removed.empty()) { + json removed = json::array(); + for (const auto &[key, ids] : frame.update->removed) { + Table table(key.second + "__removed"); + table.set("id", Column(ids)); + removed.push_back({{"dataStructure", key.first}, {"entitySet", key.second}, + {"propertyTable", add_table(table, key.first + "__" + key.second + "__removed")}}); + } + update["removed"] = removed; + } + } + + json scene = {{"version", SPECIFICATION_VERSION}, + {"frame", {{"time", frame.time}, {"timeUnit", frame.time_unit}, {"step", frame.step}}}}; + if (frame.vocabulary_version) scene["vocabulary"] = *frame.vocabulary_version; + if (!frame.writer.empty()) scene["writer"] = frame.writer; + if (!update.is_null()) scene["update"] = update; + if (part_entry) scene["part"] = part_entry(buffers); + scene["dataStructures"] = data_structures; + if (!mappings.empty()) scene["mappings"] = mappings; + if (!variables.empty()) scene["variables"] = variables; + const auto keys = frame_part_keys(frame); + if (embed_parts && !keys.empty()) { + json embedded = json::object(); + for (const auto &key : keys) embedded[key] = {{"bufferView", buffers.add_bytes(frame_part_bytes(frame, key))}}; + scene["parts"] = embedded; + } + + const json other_extensions = get(frame.unknown, "extensions", json::object()); + json extensions = {{METADATA, {{"schema", schema}, {"propertyTables", tables}}}, {EXTENSION, scene}}; + json used = json::array({METADATA, EXTENSION}); + if (!meshes.empty()) used.push_back(FEATURES); + for (const auto &item : other_extensions.items()) { + extensions[item.key()] = item.value(); + used.push_back(item.key()); + } + document["extensionsUsed"] = used; + document["extensions"] = extensions; + if (frame.unknown.is_object() && frame.unknown.contains("extras")) document["extras"] = frame.unknown["extras"]; + if (!meshes.empty()) { + document["meshes"] = meshes; + document["accessors"] = accessors; + } + // a part, or a frame of no data structure: glTF has no empty children + if (document["nodes"][0]["children"].empty()) document["nodes"][0].erase("children"); + if (!buffers.views.empty()) { + // the buffer holds the chunk's padding to 8 bytes: glTF-Validator warns of more than 3 bytes beyond it + buffers.data.resize(buffers.data.size() + detail::padding(buffers.data.size()), 0); + document["buffers"] = json::array({{{"byteLength", buffers.data.size()}}}); + document["bufferViews"] = buffers.views; + } + return pack(document, buffers.data); +} + +void write_bytes(const std::filesystem::path &path, const std::vector &bytes) { + std::ofstream file(path, std::ios::binary); + if (!file) throw Error("cannot write '" + path.string() + "'"); + file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + if (!file) throw Error("cannot write '" + path.string() + "'"); +} + +} // namespace + +std::vector write_frame(const std::string &path, const Frame &frame, bool embed_parts) { + write_bytes(path, to_bytes(frame, embed_parts)); + const auto keys = frame_part_keys(frame); + if (embed_parts) return keys; + const auto folder = std::filesystem::absolute(std::filesystem::path(path)).parent_path() / "parts"; + for (const auto &key : keys) { + const auto target = folder / (key + ".glb"); + if (std::filesystem::exists(target)) continue; // its name is its hash: its content + std::filesystem::create_directories(folder); + const auto temporary = folder / (key + ".glb.part"); + write_bytes(temporary, frame_part_bytes(frame, key)); + std::filesystem::rename(temporary, target); // a reader never sees half a part + } + return keys; +} + +std::vector frame_part_keys(const Frame &frame) { + std::set keys; + for (const auto &data_structure : frame.data_structures) + for (const auto &instance_set : data_structure.instance_sets) + keys.insert(instance_set.parts.begin(), instance_set.parts.end()); + return {keys.begin(), keys.end()}; +} + +// ── Shared parts ───────────────────────────────────────────────────────── + +std::vector part_to_bytes(const Part &part) { + if (const auto *template_part = std::get_if