diff --git a/.changeset/three-bananas-accept.md b/.changeset/three-bananas-accept.md new file mode 100644 index 0000000..7d06786 --- /dev/null +++ b/.changeset/three-bananas-accept.md @@ -0,0 +1,6 @@ +--- +'@platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering': minor +'@platforma-open/milaboratories.runenv-python-3': minor +--- + +Add Python 3.12.10 run environment for the Embedding Clustering block: `fast_hdbscan` 0.3.2 and `pynndescent` 0.6.0 on `numba` 0.67.0 / `llvmlite` 0.49.0 (0.62.1 / 0.45.1 on macOS-Intel, the last releases with a cp312 wheel there), with `numpy`, `scipy`, `scikit-learn`, `pandas`, `polars-lts-cpu` and `pyarrow`. Contrib `hdbscan` is not included: the block's B2 path uses `fast_hdbscan` only. The whole closure is pinned and installed with `noDeps`, so the seeded clustering is reproducible across rebuilds. diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6e87fb8..6b8195f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -129,6 +129,12 @@ jobs: {"os":"macos-15", "arch":"arm64", "selector":"./python-3.12.10-pgen"}, {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-pgen"}, + {"os":"ubuntu-large-arm64", "arch":"arm64", "selector":"./python-3.12.10-embedding-clustering"}, + {"os":"ubuntu-large-amd64", "arch":"amd64", "selector":"./python-3.12.10-embedding-clustering"}, + {"os":"macos-15", "arch":"arm64", "selector":"./python-3.12.10-embedding-clustering"}, + {"os":"macos-15-large", "arch":"amd64", "selector":"./python-3.12.10-embedding-clustering"}, + {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-embedding-clustering"}, + {"os":"ubuntu-large-amd64", "arch":"amd64", "selector":"./python-3.12.10-torch-cuda"}, {"os":"macos-15", "arch":"arm64", "selector":"./python-3.12.10-torch-cuda"} ] diff --git a/catalogue/package.json b/catalogue/package.json index 3d02186..bd0887e 100644 --- a/catalogue/package.json +++ b/catalogue/package.json @@ -45,6 +45,9 @@ "3.12.10-pgen": { "reference": "@platforma-open/milaboratories.runenv-python-3.12.10-pgen/dist/tengo/software/main.sw.json" }, + "3.12.10-embedding-clustering": { + "reference": "@platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering/dist/tengo/software/main.sw.json" + }, "3.10.21-clustcr": { "reference": "@platforma-open/milaboratories.runenv-python-3.10.21-clustcr/dist/tengo/software/main.sw.json" }, @@ -72,6 +75,7 @@ "@platforma-open/milaboratories.runenv-python-3.12.10-torch-cuda": "workspace:*", "@platforma-open/milaboratories.runenv-python-3.12.10-clustering": "workspace:*", "@platforma-open/milaboratories.runenv-python-3.12.10-pgen": "workspace:*", + "@platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering": "workspace:*", "@platforma-open/milaboratories.runenv-python-3.10.21-clustcr": "workspace:*", "@platforma-open/milaboratories.runenv-python-3.12.10-hilary": "workspace:*", "@platforma-open/milaboratories.runenv-python-3.12.10-antibody-design": "workspace:*" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8fe6ccd..61ea9ed 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -74,6 +74,9 @@ importers: '@platforma-open/milaboratories.runenv-python-3.12.10-clustering': specifier: workspace:* version: link:../python-3.12.10-clustering + '@platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering': + specifier: workspace:* + version: link:../python-3.12.10-embedding-clustering '@platforma-open/milaboratories.runenv-python-3.12.10-h5ad': specifier: workspace:* version: link:../python-3.12.10-h5ad @@ -166,6 +169,18 @@ importers: specifier: 'catalog:' version: 4.20.6 + python-3.12.10-embedding-clustering: + devDependencies: + '@platforma-sdk/package-builder': + specifier: 'catalog:' + version: 3.10.7 + runenv-python-builder: + specifier: workspace:* + version: link:../builder + tsx: + specifier: 'catalog:' + version: 4.20.6 + python-3.12.10-empty: devDependencies: '@platforma-sdk/package-builder': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5c1e654..6cc7c1e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -15,6 +15,7 @@ packages: - python-3.12.10-antibody-design - python-3.10.21-clustcr - python-3.12.10-pgen + - python-3.12.10-embedding-clustering - catalogue catalog: diff --git a/python-3.12.10-embedding-clustering/CHANGELOG.md b/python-3.12.10-embedding-clustering/CHANGELOG.md new file mode 100644 index 0000000..c68bac5 --- /dev/null +++ b/python-3.12.10-embedding-clustering/CHANGELOG.md @@ -0,0 +1 @@ +# @platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering diff --git a/python-3.12.10-embedding-clustering/README.md b/python-3.12.10-embedding-clustering/README.md new file mode 100644 index 0000000..a6b1fa1 --- /dev/null +++ b/python-3.12.10-embedding-clustering/README.md @@ -0,0 +1,9 @@ +Python 3.12.10 run environment for the Embedding Clustering block (`embedding-clustering`): `fast_hdbscan` and `pynndescent` on `numba`, plus `scikit-learn` (PCA, medoid distances), `polars` and `pyarrow` (Parquet loading and output) and `pandas`. + +The set is pinned flat and matches the block's `requirements.txt`. The block installs it with `pip --no-index --find-links `, so each pin in the block must exist here as a wheel. Change the pins here and in the block together. + +Every package is declared under `noDeps`, and the transitive dependencies (`joblib`, `threadpoolctl`, `python-dateutil`, `six`, `pytz`, `tzdata`) are pinned explicitly. Otherwise each package's loose requirements pull the newest `numba`, `llvmlite`, `numpy` and `scipy` next to the pinned versions. The pins keep the seeded clustering stable across rebuilds of this run environment. + +`numba` and `llvmlite` are pinned per platform under `platformSpecific`. macOS-Intel (`macosx-x64`) gets `numba` 0.62.1 / `llvmlite` 0.45.1, the last releases with a cp312 wheel for that platform; every other platform gets 0.67.0 / 0.49.0. The block's `requirements.txt` must select the same pair with environment markers, for example `numba==0.62.1; sys_platform == "darwin" and platform_machine == "x86_64"` and `numba==0.67.0; sys_platform != "darwin" or platform_machine != "x86_64"`. PEP 508 has no `not`, so the negated condition is written out with `!=` and `or`. + +The two numba versions give byte-identical clusters on the same CPU. numba compiles for the CPU it runs on by default, so clusters can differ slightly between CPUs (arm64 vs x86, or x86 CPUs with different SIMD support). The block's environment variable `NUMBA_CPU_NAME=generic` makes them identical across platforms, at a cost of about 6% in fit time. diff --git a/python-3.12.10-embedding-clustering/config.json b/python-3.12.10-embedding-clustering/config.json new file mode 100644 index 0000000..e0de46e --- /dev/null +++ b/python-3.12.10-embedding-clustering/config.json @@ -0,0 +1,76 @@ +{ + "packages": { + "dependencies": [ + "numpy==2.2.6", + "scipy==1.15.3", + "scikit-learn==1.6.1", + "joblib==1.5.3", + "threadpoolctl==3.6.0", + "pandas==2.2.3", + "python-dateutil==2.9.0.post0", + "six==1.17.0", + "pytz==2026.4", + "tzdata==2026.4", + "polars-lts-cpu==1.33.1", + "pyarrow==24.0.0", + "pynndescent==0.6.0", + "fast_hdbscan==0.3.2" + ], + "noDeps": [ + "numpy", + "scipy", + "scikit-learn", + "joblib", + "threadpoolctl", + "pandas", + "python-dateutil", + "six", + "pytz", + "tzdata", + "polars-lts-cpu", + "pyarrow", + "numba", + "llvmlite", + "pynndescent", + "fast_hdbscan" + ], + "skip": {}, + "overrides": {}, + "platformSpecific": { + "linux-x64": { + "dependencies": [ + "numba==0.67.0", + "llvmlite==0.49.0" + ] + }, + "linux-aarch64": { + "dependencies": [ + "numba==0.67.0", + "llvmlite==0.49.0" + ] + }, + "macosx-x64": { + "dependencies": [ + "numba==0.62.1", + "llvmlite==0.45.1" + ] + }, + "macosx-aarch64": { + "dependencies": [ + "numba==0.67.0", + "llvmlite==0.49.0" + ] + }, + "windows-x64": { + "dependencies": [ + "numba==0.67.0", + "llvmlite==0.49.0" + ] + } + }, + "resolution": { + "strictMissing": true + }, + "buildWheel": {} + } +} diff --git a/python-3.12.10-embedding-clustering/package.json b/python-3.12.10-embedding-clustering/package.json new file mode 100644 index 0000000..998a572 --- /dev/null +++ b/python-3.12.10-embedding-clustering/package.json @@ -0,0 +1,43 @@ +{ + "name": "@platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering", + "version": "0.1.0", + "description": "Python 3.12.10 run environment for the Embedding Clustering block (fast_hdbscan + pynndescent on numba, with scikit-learn, polars and pyarrow)", + "scripts": { + "cleanup": "rm -rf ./pkg-*.tgz ./pydist ./dist/ ./build/", + "reset": "pnpm run cleanup && rm -rf ./node_modules ./.turbo", + "build": "pl-py-builder", + "after-prebuild": "pl-pkg publish packages", + "before-publish": "pl-pkg prepublish" + }, + "files": [ + "dist/" + ], + "block-software": { + "entrypoints": { + "main": { + "environment": { + "artifact": { + "type": "environment", + "runtime": "python", + "registry": "platforma-open", + "python-version": "3.12.10", + "roots": { + "linux-x64": "./pydist/linux-x64", + "linux-aarch64": "./pydist/linux-aarch64", + "macosx-x64": "./pydist/macosx-x64", + "macosx-aarch64": "./pydist/macosx-aarch64", + "windows-x64": "./pydist/windows-x64" + }, + "binDir": "bin" + } + } + } + } + }, + "license": "UNLICENSED", + "devDependencies": { + "@platforma-sdk/package-builder": "catalog:", + "runenv-python-builder": "workspace:*", + "tsx": "catalog:" + } +}