diff --git a/docs/introduction/installation.md b/docs/introduction/installation.md index 8831a8ba..3610b0e8 100644 --- a/docs/introduction/installation.md +++ b/docs/introduction/installation.md @@ -88,9 +88,10 @@ You can find CUDA installers on [NVIDIA CUDA Toolkit Archive Website](https://de - Install [SSCMA](https://github.com/Seeed-Studio/ModelAssistant) deps ```sh +pip3 install openmim && \ pip3 install -r requirements/base.txt && \ mim install -r requirements/mmlab.txt && \ -mim install -e . +pip3 install -e . --no-deps ``` ### Step 4 - Install Extra Dependencies (Optional) @@ -124,16 +125,18 @@ Or you can do the configuration manually using Conda's configuration file. conda env create -n sscma -f environment.yml && \ conda activate sscma && \ pip3 install -r requirements/inference.txt -r requirements/export.txt -r requirements/tests.txt && \ +pip3 install openmim && \ mim install -r requirements/mmlab.txt && \ -mim install -e . +pip3 install -e . --no-deps ``` ```sh [GPU (CUDA)] conda env create -n sscma -f environment_cuda.yml && \ conda activate sscma && \ pip3 install -r requirements/inference.txt -r requirements/export.txt -r requirements/tests.txt && \ +pip3 install openmim && \ mim install -r requirements/mmlab.txt && \ -mim install -e . +pip3 install -e . --no-deps ``` ::: diff --git a/docs/zh_cn/introduction/installation.md b/docs/zh_cn/introduction/installation.md index 3ae151fb..cf6ba712 100644 --- a/docs/zh_cn/introduction/installation.md +++ b/docs/zh_cn/introduction/installation.md @@ -88,9 +88,10 @@ conda activate sscma - 安装 [SSCMA](https://github.com/Seeed-Studio/ModelAssistant) 依赖项 ```sh +pip3 install openmim && \ pip3 install -r requirements/base.txt && \ mim install -r requirements/mmlab.txt && \ -mim install -e . +pip3 install -e . --no-deps ``` ### 第 4 步 - 安装额外依赖项(可选) @@ -124,16 +125,18 @@ bash scripts/setup_linux.sh conda env create -n sscma -f environment.yml && \ conda activate sscma && \ pip3 install -r requirements/inference.txt -r requirements/export.txt -r requirements/tests.txt && \ +pip3 install openmim && \ mim install -r requirements/mmlab.txt && \ -mim install -e . +pip3 install -e . --no-deps ``` ```sh [GPU(CUDA)] conda env create -n sscma -f environment_cuda.yml && \ conda activate sscma && \ pip3 install -r requirements/inference.txt -r requirements/export.txt -r requirements/tests.txt && \ +pip3 install openmim && \ mim install -r requirements/mmlab.txt && \ -mim install -e . +pip3 install -e . --no-deps ``` ::: diff --git a/notebooks/Google-Colab-SWFIT-YOLO-A1101-Example.ipynb b/notebooks/Google-Colab-SWFIT-YOLO-A1101-Example.ipynb index dea1ebeb..9bc5f961 100644 --- a/notebooks/Google-Colab-SWFIT-YOLO-A1101-Example.ipynb +++ b/notebooks/Google-Colab-SWFIT-YOLO-A1101-Example.ipynb @@ -52,13 +52,8 @@ }, "outputs": [], "source": [ - "# Ethos-U-Vela need to be installed this way, or SSCMA does not work anymore...\n", - "!git clone https://review.mlplatform.org/ml/ethos-u/ethos-u-vela.git\n", - "%cd ethos-u-vela\n", - "!pip install .\n", - "%cd..\n", - "!git clone https://github.com/Seeed-Studio/SSCMA.git # currently we're using experimental 2.0 version branch\n", - "%cd /content/SSCMA" + "!git clone https://github.com/Seeed-Studio/ModelAssistant.git -b 2.0.0\n", + "%cd ModelAssistant" ] }, { @@ -67,7 +62,7 @@ "id": "LWAE--J9sAmW" }, "source": [ - "**Step 1:** Install python third-party library" + "**Step 1:** Install the dependencies. The setup script installs a modern PyTorch-compatible dependency stack and compiles [MMCV](https://github.com/open-mmlab/mmcv) from source (there are no prebuilt MMCV wheels for recent PyTorch/Python versions), which takes about 10-15 minutes on Colab." ] }, { @@ -78,7 +73,7 @@ }, "outputs": [], "source": [ - "!pip install ." + "!bash scripts/setup_colab.sh" ] }, { diff --git a/pyproject.toml b/pyproject.toml index e107eeb1..06356d00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,10 @@ +[build-system] +# PEP 517/660 build via setuptools. Without this, pip falls back to the +# deprecated legacy `setup.py egg_info`/`setup.py develop` path, which is +# removed in newer pip releases and fails on modern environments (e.g. Colab). +requires = ["setuptools>=64,<81", "wheel"] +build-backend = "setuptools.build_meta" + [tool.isort] profile = "black" diff --git a/requirements/base.txt b/requirements/base.txt index 3cd67992..2acf3dae 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,7 +5,12 @@ libusb1 # sensor cbor -numpy>=1.23.0,<2.0.0 +# numpy < 2.3: numba (preinstalled on Colab) requires numpy < 2.3, while +# everything else in a modern image (opencv 5, jax, ml-dtypes, tensorflow +# 2.20, h5py, keras) requires numpy >= 2. A numpy < 2.0 pin forces a numpy +# 1.26 source build on Python 3.13 and makes pip downgrade half of the +# preinstalled stack (opencv, keras, ml-dtypes, h5py, vela). +numpy>=1.23.0,<2.3.0 @@ -13,8 +18,11 @@ numpy>=1.23.0,<2.0.0 opencv-python>=4.9.0.80 -# openmmlab package manager -openmim>=0.3.7 +# NOTE: openmim is intentionally NOT listed here. It pulls in `openxlab`, +# which hard-pins `setuptools~=60.2.0` - a version too old to build anything +# on Python >= 3.12 (pkgutil.ImpImporter was removed). If you need the `mim` +# command, install it separately with `pip install openmim` and mind your +# setuptools version afterwards. packaging>=23.1 pandas>=2.0.0 pillow>=9.4.0 diff --git a/requirements/export.txt b/requirements/export.txt index 68a357f9..90ae5bb4 100644 --- a/requirements/export.txt +++ b/requirements/export.txt @@ -5,4 +5,12 @@ TinyNeuralNetwork @ https://files.seeedstudio.com/sscma/library/TinyNeuralNetwor pnnx # vela -ethos-u-vela \ No newline at end of file +# NOTE: pinned to a range with prebuilt wheels for Python >= 3.12. +# ethos-u-vela <= 4.1.0 ships sdists only and its legacy setup.py fails to +# build on Python >= 3.12 (needs flatbuffers at build time). Worse, vela +# 3.7.0 declares NO dependencies at all, so pip's resolver can always +# backtrack to it when anything else conflicts - silently turning a +# dependency conflict into a source build that then fails. +# 5.1.0 is verified with tools/export.py and the vela CLI flags used in the +# docs/notebooks; bump the ceiling after re-verifying. +ethos-u-vela>=4.2.0,<=5.1.0 \ No newline at end of file diff --git a/requirements/inference.txt b/requirements/inference.txt index ef713162..a0ab9f1a 100644 --- a/requirements/inference.txt +++ b/requirements/inference.txt @@ -1,7 +1,13 @@ libusb1>=3.0.0 -pnnx==0.0.4 +# NOTE: keep unpinned. The previous pin (==0.0.4) resolved to an ancient +# placeholder release, and it conflicts with the unpinned pnnx in +# requirements/export.txt. +pnnx ncnn>=1.0.20230517 onnx>=1.14.0 +# torch.onnx.export uses the dynamo exporter by default since PyTorch 2.9, +# which requires onnxscript +onnxscript>=0.1.0 onnxmltools>=1.11.2 onnxruntime>=1.15.1 onnxsim>=0.4.33 diff --git a/requirements/mmlab.txt b/requirements/mmlab.txt index a022d18a..020f38f0 100644 --- a/requirements/mmlab.txt +++ b/requirements/mmlab.txt @@ -1,5 +1,11 @@ -# use openmim to install -mmcls>=1.0.0.rc6 -mmcv-full<=2.1.0 -mmdet>=3.0.0, <3.1.0 # mmyolo currently does not support mmdet 3.1.0 -mmengine>=0.8.2 +# OpenMMLab dependencies +# NOTE: mmcv 2.2.0 is the last mmcv release and the only one that compiles +# against recent PyTorch (>= 2.6). There are no prebuilt mmcv wheels for +# recent PyTorch/Python combinations, so mmcv must be built from source +# (scripts/setup_colab.sh does this). mmdet 3.0.x / mmcls 1.0.0rc6 hard-code +# an mmcv < 2.1.0 ceiling at import time; use scripts/patch_mmlab_versions.py +# to relax that assertion after installation. +mmcls>=1.0.0rc6 +mmcv>=2.2.0,<=2.2.0 +mmdet>=3.0.0,<3.1.0 +mmengine>=0.8.2,<1.0.0 diff --git a/requirements/pytorch_cpu.txt b/requirements/pytorch_cpu.txt index 1052b6e9..62af8902 100644 --- a/requirements/pytorch_cpu.txt +++ b/requirements/pytorch_cpu.txt @@ -1,5 +1,9 @@ -i https://download.pytorch.org/whl/cpu -torch<=2.0.1 -torchaudio<=2.0.2 -torchvision<=0.15.2 +# NOTE: no upper version bound on purpose. The previous ceiling (torch<=2.0.1) +# has no wheels for Python >= 3.12 and made the whole installation fail. +# torch/torchvision/torchaudio versions must stay mutually compatible; +# on Google Colab the preinstalled versions are used as-is. +torch>=2.1.0 +torchaudio>=2.1.0 +torchvision>=0.16.0 diff --git a/requirements/pytorch_cuda.txt b/requirements/pytorch_cuda.txt index 990e36a2..da057cce 100644 --- a/requirements/pytorch_cuda.txt +++ b/requirements/pytorch_cuda.txt @@ -1,5 +1,9 @@ --i https://download.pytorch.org/whl/cu118 - -torch<=2.0.1 -torchaudio<=2.0.2 -torchvision<=0.15.2 +# NOTE: no upper version bound on purpose. The previous ceiling (torch<=2.0.1) +# has no wheels for Python >= 3.12 and made the whole installation fail. +# Modern PyPI torch wheels already bundle the CUDA runtime, so no custom +# index URL is required anymore. torch/torchvision/torchaudio versions must +# stay mutually compatible; on Google Colab the preinstalled versions are +# used as-is. +torch>=2.1.0 +torchaudio>=2.1.0 +torchvision>=0.16.0 diff --git a/scripts/patch_mmlab_versions.py b/scripts/patch_mmlab_versions.py new file mode 100755 index 00000000..1cbcc5aa --- /dev/null +++ b/scripts/patch_mmlab_versions.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# Copyright (c) Seeed Technology Co.,Ltd. All rights reserved. +"""Relax the hard-coded MMCV upper-version assertions of mmdet/mmcls. + +mmdet 3.0.x and mmcls 1.0.0rc6 assert ``mmcv < 2.1.0`` at import time. +We deliberately use mmcv 2.2.0 - the final mmcv release and the only one +that compiles against recent PyTorch (>= 2.6) - so the assertion must be +relaxed in the *installed* packages. + +The patch is applied in-place to the installed ``__init__.py`` files and +fails loudly if the expected pattern is not found (e.g. the package +versions drifted), instead of silently doing nothing. + +Usage: + python scripts/patch_mmlab_versions.py # patch + python scripts/patch_mmlab_versions.py --verify # patch + import check +""" + +import argparse +import importlib.util +import os +import re +import sys + +# packages -> new mmcv maximum version (mmcv 3.x does not exist, so this +# effectively disables the upper bound while remaining an explicit bound) +PACKAGES = { + 'mmdet': '3.0.0', + 'mmcls': '3.0.0', +} + +PATTERN = re.compile(r"mmcv_maximum_version\s*=\s*['\"][\d.]+['\"]") + + +def patch_package(name: str, new_maximum: str) -> bool: + spec = importlib.util.find_spec(name) + if spec is None or not spec.submodule_search_locations: + print(f'[SKIP] {name} is not installed') + return False + + init_file = os.path.join(spec.submodule_search_locations[0], '__init__.py') + with open(init_file, 'r', encoding='utf-8') as f: + content = f.read() + + replacement = f"mmcv_maximum_version = '{new_maximum}'" + patched, count = PATTERN.subn(replacement, content) + + if count == 0: + if f"mmcv_maximum_version = '{new_maximum}'" in content: + print(f'[OK] {name}: already patched ({init_file})') + return True + print(f'[FAIL] {name}: no mmcv_maximum_version found in {init_file}') + print(' The installed version may have changed - please check it manually.') + return False + + with open(init_file, 'w', encoding='utf-8') as f: + f.write(patched) + print(f'[OK] {name}: mmcv_maximum_version -> {new_maximum} ({init_file})') + return True + + +ADAFACTOR_OLD = " OPTIMIZERS.register_module(name='Adafactor', module=Adafactor)\n transformer_optimizers.append('Adafactor')" +ADAFACTOR_NEW = ( + " if 'Adafactor' not in OPTIMIZERS:\n" + " OPTIMIZERS.register_module(name='Adafactor', module=Adafactor)\n" + " transformer_optimizers.append('Adafactor')" +) + + +def patch_mmengine_adafactor() -> bool: + """Guard mmengine's transformers-Adafactor registration. + + mmengine (<= 0.10.7, including the current upstream) unconditionally + registers transformers' Adafactor when transformers is installed. Since + torch >= 2.9 ships torch.optim.Adafactor - already registered under the + same name - importing mmengine.optim crashes with + ``KeyError: 'Adafactor is already registered in optimizer ...'`` whenever + both torch >= 2.9 and transformers are present (both are preinstalled on + Colab). + """ + spec = importlib.util.find_spec('mmengine') + if spec is None or not spec.submodule_search_locations: + print('[SKIP] mmengine is not installed') + return False + + builder_file = os.path.join( + spec.submodule_search_locations[0], 'optim', 'optimizer', 'builder.py' + ) + with open(builder_file, 'r', encoding='utf-8') as f: + content = f.read() + + if ADAFACTOR_NEW in content: + print(f'[OK] mmengine: Adafactor guard already present ({builder_file})') + return True + if ADAFACTOR_OLD not in content: + print(f'[FAIL] mmengine: Adafactor registration pattern not found in {builder_file}') + print(' The installed mmengine version may have changed - please check it manually.') + return False + + with open(builder_file, 'w', encoding='utf-8') as f: + f.write(content.replace(ADAFACTOR_OLD, ADAFACTOR_NEW)) + print(f'[OK] mmengine: guarded Adafactor registration ({builder_file})') + return True + + +def verify_imports() -> bool: + ok = True + for name in ('mmcv', 'mmdet', 'mmcls', 'mmengine'): + try: + module = __import__(name) + print(f'[OK] import {name} {module.__version__}') + except Exception as exc: # noqa: BLE001 + print(f'[FAIL] import {name}: {exc}') + ok = False + if ok: + try: + # triggers the optimizer registration code paths + from mmengine.optim.optimizer import OPTIMIZERS # noqa: F401 + + print('[OK] mmengine optimizer registry builds') + except Exception as exc: # noqa: BLE001 + print(f'[FAIL] mmengine optimizer registry: {exc}') + ok = False + return ok + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--verify', action='store_true', help='also verify the patched packages can be imported') + args = parser.parse_args() + + results = [patch_package(name, maximum) for name, maximum in PACKAGES.items()] + results.append(patch_mmengine_adafactor()) + if not all(results): + sys.exit(1) + + if args.verify and not verify_imports(): + sys.exit(1) + + print('Done.') + + +if __name__ == '__main__': + main() diff --git a/scripts/setup_colab.sh b/scripts/setup_colab.sh index 97f70d53..f9e2aa29 100755 --- a/scripts/setup_colab.sh +++ b/scripts/setup_colab.sh @@ -1,8 +1,17 @@ #!/bin/bash +# Setup script for Google Colab (also works on plain Linux with NVIDIA GPU). +# +# Order matters: +# 1. torch/torchvision/torchaudio must be installed FIRST and must not be +# replaced afterwards - the mmcv extension is compiled against them. +# (Some dependencies, e.g. pyvww, pull an unpinned `torchvision`, whose +# latest release pins an exact torch version and would silently swap it.) +# 2. numpy is pinned (< 2.3, for numba) BEFORE anything is compiled. +# 3. mmcv is compiled from source LAST, after all pip installs are done. +set -e - -# configs -CUDA_AVAILABLE="$(command -v nvidia-smi)" +# always operate from the repository root, regardless of the caller's cwd +cd "$(dirname "${BASH_SOURCE[0]}")/.." # ansi colors @@ -11,38 +20,142 @@ GREEN='\033[032m' BLUE='\033[034m' RST='\033[m' + # check cuda echo -en "Checking if CUDA available... " -if [ ! "${CUDA_AVAILABLE}" ]; then +if [ ! "$(command -v nvidia-smi)" ]; then echo -en "${RED}Not found!${RST}\n" - echo -en "Please enable GPU Runtime${RST}\n" + echo -en "Please enable the GPU runtime (Runtime -> Change runtime type)${RST}\n" exit 1 else echo -en "${GREEN}OK${RST}\n" fi -# limit torch version -pip install torch==2.0.0 torchvision==0.15.1 torchaudio==2.0.1 - -# install base deps -echo -en "Installing base deps... " -pip install -r requirements/base.txt -r requirements/inference.txt -r requirements/export.txt -r requirements/tests.txt -if [ "$?" != 0 ]; then - echo -en "Install base deps failed... ${RED}Exiting${RST}\n" - exit 1 +# step 1: ensure a modern pytorch stack; on Colab the preinstalled one is used +echo -en "${BLUE}Ensuring PyTorch stack... ${RST}\n" +if ! python -c "import torch, torchvision, torchaudio" > /dev/null 2>&1; then + pip install torch torchvision torchaudio fi +python -c "import torch, torchvision, torchaudio; print(f'torch={torch.__version__} torchvision={torchvision.__version__} torchaudio={torchaudio.__version__}')" +# Pin the torch stack for every subsequent pip install: pip's resolver does +# not respect locally-versioned (+cpu/+cuXXX) installed packages and would +# otherwise happily replace them (e.g. pnnx depends on plain `torch`), which +# breaks the mmcv build and the CUDA/driver match. +CONSTRAINTS_FILE="$(mktemp)" +trap 'rm -f "${CONSTRAINTS_FILE}"' EXIT +python -c "import torch, torchvision, torchaudio; print(f'torch=={torch.__version__}'); print(f'torchvision=={torchvision.__version__}'); print(f'torchaudio=={torchaudio.__version__}')" > "${CONSTRAINTS_FILE}" +cat "${CONSTRAINTS_FILE}" -# openmim install deps -echo -en "Installing OpenMIM deps... \n" -mim install -r requirements/mmlab.txt && \ -mim install mmcv==2.0.0 && \ -mim install -e . -if [ "$?" != 0 ]; then - echo -en "OpenMIM install deps failed... ${RED}Exiting${RST}\n" - exit 1 -fi + +# step 2: build tools and pinned numpy (must happen before compiling mmcv) +echo -en "${BLUE}Installing build tools... ${RST}\n" +# setuptools<81: still ships the distutils shim required by TinyNeuralNetwork +pip install "numpy>=1.23.0,<2.3.0" "setuptools>=49.4.0,<81" Cython ninja wheel packaging + + +# step 3: python dependencies (with the torch stack constrained, nothing here +# can pull a different torch version) +echo -en "${BLUE}Installing base deps... ${RST}\n" +pip install -c "${CONSTRAINTS_FILE}" -r requirements/base.txt -r requirements/inference.txt -r requirements/export.txt -r requirements/tests.txt + + +# step 4: OpenMMLab deps (pip metadata of mmdet/mmcls does not pull mmcv, so +# this cannot interfere with the mmcv source build below) +echo -en "${BLUE}Installing OpenMMLab deps... ${RST}\n" +pip install -c "${CONSTRAINTS_FILE}" "mmengine>=0.8.2,<1.0.0" "mmdet>=3.0.0,<3.1.0" "mmcls>=1.0.0rc6" + +# vela: wheel-only. Sdist releases (<= 4.1.0) fail to build on Python >= 3.12 +# (their setup.py imports flatbuffers at build time), and the metadata-less +# vela 3.7.0 is pip's favorite backtracking target on conflict-heavy +# environments like Colab. +pip install --only-binary ethos-u-vela "ethos-u-vela>=4.2.0,<=5.1.0" + + +# step 5: build mmcv from source - there are no prebuilt wheels for recent +# PyTorch/Python versions. Compiling with CUDA ops takes ~10-15 min on Colab. +# MAX_JOBS is capped at 4 by default: each nvcc job needs ~1-2 GB RAM and +# Colab instances only have ~12 GB. Override with MAX_JOBS=N on bigger hosts. +# +# The sdist is downloaded and patched BEFORE building: mmcv 2.2.0's setup.py +# get_version() does exec(...) at function scope and reads locals(), which no +# longer works on Python >= 3.13 (PEP 667) and kills the build with +# "KeyError: '__version__'" at the metadata step. It cannot be fetched with +# pip (neither pip install nor pip download): pip ALWAYS builds the sdist's +# metadata first - i.e. runs the broken setup.py before we can patch it - so +# the sdist is fetched from PyPI out-of-band (with sha256 verification). +echo -en "${BLUE}Building mmcv 2.2.0 from source (this takes a while)... ${RST}\n" +MMCV_BUILD_DIR="$(mktemp -d)" +trap 'rm -rf "${CONSTRAINTS_FILE}" "${MMCV_BUILD_DIR}"' EXIT +read -r MMCV_URL MMCV_SHA256 < <(python - <<'EOF' +import json +import urllib.request + +with urllib.request.urlopen('https://pypi.org/pypi/mmcv/2.2.0/json') as r: + data = json.load(r) +sdist = next(u for u in data['urls'] if u['packagetype'] == 'sdist') +print(sdist['url'], sdist['digests']['sha256']) +EOF +) +curl -fSL --retry 3 -o "${MMCV_BUILD_DIR}/mmcv-2.2.0.tar.gz" "${MMCV_URL}" +echo "${MMCV_SHA256} ${MMCV_BUILD_DIR}/mmcv-2.2.0.tar.gz" | sha256sum -c - +tar xzf "${MMCV_BUILD_DIR}/mmcv-2.2.0.tar.gz" -C "${MMCV_BUILD_DIR}" +python - "${MMCV_BUILD_DIR}/mmcv-2.2.0/setup.py" <<'EOF' +import sys + +path = sys.argv[1] +src = open(path).read() +old = ( + " with open(version_file, encoding='utf-8') as f:\n" + " exec(compile(f.read(), version_file, 'exec'))\n" + " return locals()['__version__']" +) +new = ( + " namespace = {}\n" + " with open(version_file, encoding='utf-8') as f:\n" + " exec(compile(f.read(), version_file, 'exec'), namespace)\n" + " return namespace['__version__']" +) +if old not in src: + if 'namespace' in src and "namespace['__version__']" in src: + print('mmcv setup.py already py3.13-compatible, nothing to patch') + sys.exit(0) + print('ERROR: could not find get_version() exec pattern in mmcv setup.py -') + print('mmcv may have changed; please report this.') + sys.exit(1) +open(path, 'w').write(src.replace(old, new)) +print('patched mmcv setup.py get_version() for Python >= 3.13') +EOF +MMCV_WITH_OPS=1 MAX_JOBS="${MAX_JOBS:-4}" pip install --no-build-isolation --no-cache-dir "${MMCV_BUILD_DIR}/mmcv-2.2.0" + + +# step 6: relax the mmcv < 2.1.0 assertion hard-coded in mmdet/mmcls +echo -en "${BLUE}Patching OpenMMLab version checks... ${RST}\n" +python scripts/patch_mmlab_versions.py --verify + + +# step 7: install sscma itself (deps were installed above; --no-deps avoids +# re-resolving legacy pins) +echo -en "${BLUE}Installing sscma... ${RST}\n" +pip install --no-deps -e . + + +# step 8: smoke test +echo -en "${BLUE}Running smoke test... ${RST}\n" +python - <<'EOF' +import torch +import mmcv +import mmdet +import mmcls +import mmengine +import sscma.datasets, sscma.engine, sscma.evaluation, sscma.models, sscma.visualization +from mmcv.ops import nms + +keep = nms(torch.rand(10, 4), torch.rand(10), 0.5) +print(f'sscma={sscma.__version__} mmcv={mmcv.__version__} mmdet={mmdet.__version__} ' + f'mmcls={mmcls.__version__} mmengine={mmengine.__version__} torch={torch.__version__}') +print('mmcv ops (nms) work:', len(keep[0]) > 0) +EOF echo -en "Finished setup... ${GREEN}OK${RST}\n" -exit 0 diff --git a/scripts/setup_docker.sh b/scripts/setup_docker.sh index 6bf4f3c6..09e8544e 100755 --- a/scripts/setup_docker.sh +++ b/scripts/setup_docker.sh @@ -49,9 +49,13 @@ fi # openmim install deps +# NOTE: openmim is no longer part of requirements/base.txt (it drags in a +# setuptools pin that breaks Python >= 3.12); install it explicitly and keep +# a working setuptools afterwards. echo -en "Installing OpenMIM deps... \n" +${PYTHON_PATH} -m pip install openmim "setuptools>=69,<81" && \ ${PYTHON_PATH} -m mim install -r requirements/mmlab.txt && \ -${PYTHON_PATH} -m mim install -e . +${PYTHON_PATH} -m pip install -e . --no-deps if [ "$?" != 0 ]; then echo -en "OpenMIM install deps failed... ${RED}Exiting${RST}\n" exit 1 diff --git a/scripts/setup_linux.sh b/scripts/setup_linux.sh index 795c1f72..cfcf62c4 100755 --- a/scripts/setup_linux.sh +++ b/scripts/setup_linux.sh @@ -67,9 +67,13 @@ fi # openmim install deps +# NOTE: openmim is no longer part of requirements/base.txt (it drags in a +# setuptools pin that breaks Python >= 3.12); install it explicitly and keep +# a working setuptools afterwards. echo -en "Installing OpenMIM deps... \n" +conda run -n sscma pip install openmim "setuptools>=69,<81" && \ conda run -n sscma mim install -r requirements/mmlab.txt && \ -conda run -n sscma mim install -e . +conda run -n sscma pip install -e . --no-deps if [ "$?" != 0 ]; then echo -en "OpenMIM install deps failed... ${RED}Exiting${RST}\n" exit 1 diff --git a/setup.py b/setup.py index 840ef7c9..5128e2c0 100644 --- a/setup.py +++ b/setup.py @@ -15,10 +15,13 @@ def readme(): def get_version(): + # NOTE: exec into an explicit namespace dict - writing to function + # locals() via exec no longer works on Python >= 3.13 + namespace = {} version_file = 'sscma/version.py' with open(version_file, 'r', encoding='utf-8') as f: - exec(compile(f.read(), version_file, 'exec')) - return locals()['__version__'] + exec(compile(f.read(), version_file, 'exec'), namespace) + return namespace['__version__'] def parse_line(line: str): @@ -47,7 +50,17 @@ def parse_requirements(fpath: str = ''): line = line.strip() if line and not line.startswith('#'): if line.startswith('-r'): - res = parse_requirements(line.split(' ')[-1]) + target = line.split(' ')[-1] + # OpenMMLab packages (mmcv/mmdet/mmcls/mmengine) are + # environment-sensitive: mmcv has no prebuilt wheels for + # recent PyTorch/Python and must be built from source with + # MMCV_WITH_OPS=1. They must NOT end up in install_requires, + # otherwise `pip install .` would silently build a CPU-only + # (or legacy mmcv-full) mmcv and break the installation. + # Install them with scripts/setup_colab.sh or manually. + if target.endswith('mmlab.txt'): + continue + res = parse_requirements(target) reqs += res[0] index += res[1] elif line.startswith('-i'): diff --git a/sscma/datasets/utils/download.py b/sscma/datasets/utils/download.py index 447c98d6..2d4b9fe3 100644 --- a/sscma/datasets/utils/download.py +++ b/sscma/datasets/utils/download.py @@ -158,7 +158,7 @@ def check_file(path, store_dir=None, data_name=None): download_dir = ( f"{os.environ['HOME']}/datasets" if platform.system() == 'Linux' and not store_dir - else 'D:\datasets' + else r'D:\datasets' if not store_dir else store_dir ) @@ -175,7 +175,7 @@ def check_file(path, store_dir=None, data_name=None): download_dir = ( f"{os.environ['HOME']}/datasets" if platform.system() == 'Linux' and not store_dir - else 'D:\datasets' + else r'D:\datasets' if not store_dir else store_dir ) diff --git a/tools/export.py b/tools/export.py index bf22effb..ce92ba6c 100644 --- a/tools/export.py +++ b/tools/export.py @@ -6,6 +6,13 @@ import tempfile import torch + +# PyTorch >= 2.6 changed the default of `torch.load` to `weights_only=True`, +# which rejects checkpoints containing non-tensor objects (e.g. NumPy arrays +# in `meta`) and breaks mmengine's checkpoint loading. Restore the previous +# behavior for our trusted checkpoints; can be overridden from outside. +os.environ.setdefault('TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD', '1') + from tqdm import tqdm current_path = osp.dirname(osp.abspath(__file__)) @@ -291,6 +298,15 @@ def export_pnnx(args, model): import sys import pnnx + + if not hasattr(pnnx, 'wrapper'): + raise RuntimeError( + f'The installed pnnx=={getattr(pnnx, "__version__", "unknown")} no longer provides ' + '"pnnx.wrapper" (removed in pnnx >= 20231211, which also switched from TorchScript to ' + 'torch.export). The "pnnx" export target is currently incompatible with modern pnnx ' + 'releases - please export with --targets tflite/onnx/vela instead.' + ) + from pnnx.wrapper import convert_inputshape_to_cmd model.eval() @@ -497,13 +513,27 @@ def main(): model = runner.model.to(device=args.device) loader = runner.val_dataloader + failed_targets = [] for target in args.targets: - if target == 'tflite': - export_tflite(args, model, loader) - elif target == 'onnx': - export_onnx(args, model) - elif target == 'pnnx': - export_pnnx(args, model) + # Export targets are independent: a failure in one (e.g. pnnx, whose + # upstream package changed its API) must not discard the artifacts + # already produced by the others (e.g. tflite/vela). + try: + if target == 'tflite': + export_tflite(args, model, loader) + elif target == 'onnx': + export_onnx(args, model) + elif target == 'pnnx': + export_pnnx(args, model) + except Exception as exc: + failed_targets.append(target) + print(f'ERROR: exporting target "{target}" failed: {exc}') + + if failed_targets: + raise RuntimeError( + f'Export failed for target(s): {failed_targets}. ' + 'Artifacts for the remaining targets were exported successfully.' + ) if __name__ == '__main__': diff --git a/tools/inference.py b/tools/inference.py index 00430aba..5638dedc 100644 --- a/tools/inference.py +++ b/tools/inference.py @@ -7,6 +7,12 @@ import torch +# PyTorch >= 2.6 changed the default of `torch.load` to `weights_only=True`, +# which rejects checkpoints containing non-tensor objects (e.g. NumPy arrays +# in `meta`) and breaks mmengine's checkpoint loading. Restore the previous +# behavior for our trusted checkpoints; can be overridden from outside. +os.environ.setdefault('TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD', '1') + current_path = osp.dirname(osp.abspath(__file__)) sys.path.append(osp.dirname(current_path)) diff --git a/tools/train.py b/tools/train.py index 8d755b7b..33e4b4f3 100644 --- a/tools/train.py +++ b/tools/train.py @@ -7,6 +7,12 @@ import torch +# PyTorch >= 2.6 changed the default of `torch.load` to `weights_only=True`, +# which rejects checkpoints containing non-tensor objects (e.g. NumPy arrays +# in `meta`) and breaks mmengine's checkpoint loading. Restore the previous +# behavior for our trusted checkpoints; can be overridden from outside. +os.environ.setdefault('TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD', '1') + current_path = osp.dirname(osp.abspath(__file__)) sys.path.append(osp.dirname(current_path)) diff --git a/tools/utils/quant_read.py b/tools/utils/quant_read.py index 883ec781..813f2596 100644 --- a/tools/utils/quant_read.py +++ b/tools/utils/quant_read.py @@ -1,12 +1,45 @@ # Copyright (c) Seeed Technology Co.,Ltd. All rights reserved. import os +import random import numpy as np from onnxruntime.quantization import CalibrationDataReader from PIL import Image from torchvision.transforms import Compose, Grayscale, Resize, ToTensor -img_format = ['.JPG', '.PNG', '.JPEG'] +IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp') + + +def find_and_sample_images(folder_path, limit=10000, sample_size=100): + """Recursively find images in ``folder_path`` and randomly sample them. + + Backported from main (dea366b, issue #294): a flat ``os.listdir`` with a + narrow extension filter silently finds zero images when the calibration + folder contains subdirectories or other image formats, which then + produces a broken quantized model without any error. + """ + image_files = [] + + if not os.path.isdir(folder_path): + raise ValueError(f"The provided image path '{folder_path}' is not a valid directory.") + + for root, _, files in os.walk(folder_path): + for file in files: + if os.path.splitext(file)[1].lower() in IMG_EXTENSIONS: + image_files.append(os.path.join(root, file)) + if len(image_files) >= limit: + break + if len(image_files) >= limit: + break + + found = len(image_files) + if found < sample_size: + if found == 0: + raise ValueError(f"No images found in the directory '{folder_path}'.") + print(f"Warning: Found only {found} images, which is less than the requested sample size of {sample_size}.") + sample_size = found + + return random.sample(image_files, sample_size) class Quan_Reader(CalibrationDataReader): @@ -22,10 +55,7 @@ def __init__(self, images_folder, size, input_name, batch_size=1) -> None: self.init() def init(self): - file_ls = os.listdir(self.images_folder) - self.file_ls = iter( - [os.path.join(self.images_folder, i) for i in file_ls if os.path.splitext(i)[-1].upper() in img_format] - ) + self.file_ls = iter(find_and_sample_images(self.images_folder, limit=10000, sample_size=10000)) def get_next(self) -> dict: try: