New env for embeddings clustering with fast_hdbscan

.changeset/three-bananas-accept.md

@@ -0,0 +1,6 @@
+---
+'@platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering': minor
+'@platforma-open/milaboratories.runenv-python-3': minor
+---
+
+Add Python 3.12.10 run environment for the Embedding Clustering block: `fast_hdbscan` 0.3.2 and `pynndescent` 0.6.0 on `numba` 0.67.0 / `llvmlite` 0.49.0 (0.62.1 / 0.45.1 on macOS-Intel, the last releases with a cp312 wheel there), with `numpy`, `scipy`, `scikit-learn`, `pandas`, `polars-lts-cpu` and `pyarrow`. Contrib `hdbscan` is not included: the block's B2 path uses `fast_hdbscan` only. The whole closure is pinned and installed with `noDeps`, so the seeded clustering is reproducible across rebuilds.

.github/workflows/build.yaml

@@ -129,6 +129,12 @@ jobs:
           {"os":"macos-15", "arch":"arm64", "selector":"./python-3.12.10-pgen"},
           {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-pgen"},
 
+          {"os":"ubuntu-large-arm64", "arch":"arm64", "selector":"./python-3.12.10-embedding-clustering"},
+          {"os":"ubuntu-large-amd64", "arch":"amd64", "selector":"./python-3.12.10-embedding-clustering"},
+          {"os":"macos-15", "arch":"arm64", "selector":"./python-3.12.10-embedding-clustering"},
+          {"os":"macos-15-large", "arch":"amd64", "selector":"./python-3.12.10-embedding-clustering"},
+          {"os":"windows-latest", "arch":"amd64", "selector":"./python-3.12.10-embedding-clustering"},
+
           {"os":"ubuntu-large-amd64", "arch":"amd64", "selector":"./python-3.12.10-torch-cuda"},
           {"os":"macos-15", "arch":"arm64", "selector":"./python-3.12.10-torch-cuda"}
         ]

catalogue/package.json

@@ -45,6 +45,9 @@
       "3.12.10-pgen": {
         "reference": "@platforma-open/milaboratories.runenv-python-3.12.10-pgen/dist/tengo/software/main.sw.json"
       },
+      "3.12.10-embedding-clustering": {
+        "reference": "@platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering/dist/tengo/software/main.sw.json"
+      },
       "3.10.21-clustcr": {
         "reference": "@platforma-open/milaboratories.runenv-python-3.10.21-clustcr/dist/tengo/software/main.sw.json"
       },
@@ -72,6 +75,7 @@
     "@platforma-open/milaboratories.runenv-python-3.12.10-torch-cuda": "workspace:*",
     "@platforma-open/milaboratories.runenv-python-3.12.10-clustering": "workspace:*",
     "@platforma-open/milaboratories.runenv-python-3.12.10-pgen": "workspace:*",
+    "@platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering": "workspace:*",
     "@platforma-open/milaboratories.runenv-python-3.10.21-clustcr": "workspace:*",
     "@platforma-open/milaboratories.runenv-python-3.12.10-hilary": "workspace:*",
     "@platforma-open/milaboratories.runenv-python-3.12.10-antibody-design": "workspace:*"

pnpm-workspace.yaml

@@ -15,6 +15,7 @@ packages:
   - python-3.12.10-antibody-design
   - python-3.10.21-clustcr
   - python-3.12.10-pgen
+  - python-3.12.10-embedding-clustering
   - catalogue
 
 catalog:

python-3.12.10-embedding-clustering/CHANGELOG.md

@@ -0,0 +1 @@
+# @platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering

python-3.12.10-embedding-clustering/README.md

@@ -0,0 +1,9 @@
+Python 3.12.10 run environment for the Embedding Clustering block (`embedding-clustering`): `fast_hdbscan` and `pynndescent` on `numba`, plus `scikit-learn` (PCA, medoid distances), `polars` and `pyarrow` (Parquet loading and output) and `pandas`.
+
+The set is pinned flat and matches the block's `requirements.txt`. The block installs it with `pip --no-index --find-links <runenv packages>`, so each pin in the block must exist here as a wheel. Change the pins here and in the block together.
+
+Every package is declared under `noDeps`, and the transitive dependencies (`joblib`, `threadpoolctl`, `python-dateutil`, `six`, `pytz`, `tzdata`) are pinned explicitly. Otherwise each package's loose requirements pull the newest `numba`, `llvmlite`, `numpy` and `scipy` next to the pinned versions. The pins keep the seeded clustering stable across rebuilds of this run environment.
+
+`numba` and `llvmlite` are pinned per platform under `platformSpecific`. macOS-Intel (`macosx-x64`) gets `numba` 0.62.1 / `llvmlite` 0.45.1, the last releases with a cp312 wheel for that platform; every other platform gets 0.67.0 / 0.49.0. The block's `requirements.txt` must select the same pair with environment markers, for example `numba==0.62.1; sys_platform == "darwin" and platform_machine == "x86_64"` and `numba==0.67.0; sys_platform != "darwin" or platform_machine != "x86_64"`. PEP 508 has no `not`, so the negated condition is written out with `!=` and `or`.
+
+The two numba versions give byte-identical clusters on the same CPU. numba compiles for the CPU it runs on by default, so clusters can differ slightly between CPUs (arm64 vs x86, or x86 CPUs with different SIMD support). The block's environment variable `NUMBA_CPU_NAME=generic` makes them identical across platforms, at a cost of about 6% in fit time.

python-3.12.10-embedding-clustering/config.json

@@ -0,0 +1,76 @@
+{
+  "packages": {
+    "dependencies": [
+      "numpy==2.2.6",
+      "scipy==1.15.3",
+      "scikit-learn==1.6.1",
+      "joblib==1.5.3",
+      "threadpoolctl==3.6.0",
+      "pandas==2.2.3",
+      "python-dateutil==2.9.0.post0",
+      "six==1.17.0",
+      "pytz==2026.4",
+      "tzdata==2026.4",
+      "polars-lts-cpu==1.33.1",
+      "pyarrow==24.0.0",
+      "pynndescent==0.6.0",
+      "fast_hdbscan==0.3.2"
+    ],
+    "noDeps": [
+      "numpy",
+      "scipy",
+      "scikit-learn",
+      "joblib",
+      "threadpoolctl",
+      "pandas",
+      "python-dateutil",
+      "six",
+      "pytz",
+      "tzdata",
+      "polars-lts-cpu",
+      "pyarrow",
+      "numba",
+      "llvmlite",
+      "pynndescent",
+      "fast_hdbscan"
+    ],
+    "skip": {},
+    "overrides": {},
+    "platformSpecific": {
+      "linux-x64": {
+        "dependencies": [
+          "numba==0.67.0",
+          "llvmlite==0.49.0"
+        ]
+      },
+      "linux-aarch64": {
+        "dependencies": [
+          "numba==0.67.0",
+          "llvmlite==0.49.0"
+        ]
+      },
+      "macosx-x64": {
+        "dependencies": [
+          "numba==0.62.1",
+          "llvmlite==0.45.1"
+        ]
+      },
+      "macosx-aarch64": {
+        "dependencies": [
+          "numba==0.67.0",
+          "llvmlite==0.49.0"
+        ]
+      },
+      "windows-x64": {
+        "dependencies": [
+          "numba==0.67.0",
+          "llvmlite==0.49.0"
+        ]
+      }
+    },
+    "resolution": {
+      "strictMissing": true
+    },
+    "buildWheel": {}
+  }
+}

python-3.12.10-embedding-clustering/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@platforma-open/milaboratories.runenv-python-3.12.10-embedding-clustering",
+  "version": "0.1.0",
+  "description": "Python 3.12.10 run environment for the Embedding Clustering block (fast_hdbscan + pynndescent on numba, with scikit-learn, polars and pyarrow)",
+  "scripts": {
+    "cleanup": "rm -rf ./pkg-*.tgz ./pydist ./dist/ ./build/",
+    "reset": "pnpm run cleanup && rm -rf ./node_modules ./.turbo",
+    "build": "pl-py-builder",
+    "after-prebuild": "pl-pkg publish packages",
+    "before-publish": "pl-pkg prepublish"
+  },
+  "files": [
+    "dist/"
+  ],
+  "block-software": {
+    "entrypoints": {
+      "main": {
+        "environment": {
+          "artifact": {
+            "type": "environment",
+            "runtime": "python",
+            "registry": "platforma-open",
+            "python-version": "3.12.10",
+            "roots": {
+              "linux-x64": "./pydist/linux-x64",
+              "linux-aarch64": "./pydist/linux-aarch64",
+              "macosx-x64": "./pydist/macosx-x64",
+              "macosx-aarch64": "./pydist/macosx-aarch64",
+              "windows-x64": "./pydist/windows-x64"
+            },
+            "binDir": "bin"
+          }
+        }
+      }
+    }
+  },
+  "license": "UNLICENSED",
+  "devDependencies": {
+    "@platforma-sdk/package-builder": "catalog:",
+    "runenv-python-builder": "workspace:*",
+    "tsx": "catalog:"
+  }
+}