From 3de2a56e5ec820e785a66835433dd897d99aff2b Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 10 Sep 2026 08:36:27 +0000 Subject: [PATCH] fix(openfold3): load the selected product runtime The separately built qualification executable searches its native build for libtrtmc_runtime.so while loading backend and family plugins from the selected wheel. Those product builds can differ, so the runtime loader rejects the backend before structure prediction starts. Resolve the selected core's companion loader and prepend that library directory to the qualification subprocess's search path. Handle both copied and symlinked staged roots without changing the parent environment or any prediction acceptance criteria. Add a compiled qualification regression with two runtime/core/backend builds and a missing-loader check. Both layouts fail before the fix and pass afterward; all 20 selected OpenFold3 CPU tests pass. A separate probe also verifies the real product-build guard. Full checkpoint GPU E2E remains to be validated. Signed-off-by: chaofengw --- .../tests/cpp/fake_build_identity.cpp | 8 ++ .../tests/cpp/fake_qualification_runtime.cpp | 44 +++++++ families/openfold3/tests/test_e2e.py | 11 ++ families/openfold3/tests/test_e2e_runtime.py | 123 ++++++++++++++++++ 4 files changed, 186 insertions(+) create mode 100644 families/openfold3/tests/cpp/fake_build_identity.cpp create mode 100644 families/openfold3/tests/cpp/fake_qualification_runtime.cpp create mode 100644 families/openfold3/tests/test_e2e_runtime.py diff --git a/families/openfold3/tests/cpp/fake_build_identity.cpp b/families/openfold3/tests/cpp/fake_build_identity.cpp new file mode 100644 index 000000000..3a9b524c0 --- /dev/null +++ b/families/openfold3/tests/cpp/fake_build_identity.cpp @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +extern "C" int openfold3_test_build_id() { + return TEST_BUILD_ID; +} diff --git a/families/openfold3/tests/cpp/fake_qualification_runtime.cpp b/families/openfold3/tests/cpp/fake_qualification_runtime.cpp new file mode 100644 index 000000000..3d858ebe7 --- /dev/null +++ b/families/openfold3/tests/cpp/fake_qualification_runtime.cpp @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "trtmc/openfold3/structure_prediction.h" +#include "trtmc/runtime/family_loader.h" + +#include +#include +#include + +extern "C" int openfold3_test_build_id(); + +namespace { + +class Prediction final : public trtmc::openfold3::IStructurePrediction { + public: + trtmc::openfold3::StructurePredictionResult predict_structure(const std::string&) override { + return {"data_test\n", "{\"build_id\":" + std::to_string(TEST_BUILD_ID) + "}"}; + } +}; + +} // namespace + +namespace trtmc { + +std::unique_ptr load_task(const std::string&, const std::string& runtime_root, std::uint64_t, + const std::string&, bool) { + const auto backend = std::filesystem::path(runtime_root) / "libtrtmc_backend_trt.so"; + void* library = dlopen(backend.c_str(), RTLD_NOW | RTLD_LOCAL); + if (library == nullptr) + throw std::runtime_error(dlerror()); + const auto backend_build = + reinterpret_cast(dlsym(library, "openfold3_test_build_id")); + const bool matches = backend_build != nullptr && backend_build() == TEST_BUILD_ID && + openfold3_test_build_id() == TEST_BUILD_ID; + dlclose(library); + if (!matches) + throw std::runtime_error("qualification runtime/core/backend product build mismatch"); + return std::make_unique(); +} + +} // namespace trtmc diff --git a/families/openfold3/tests/test_e2e.py b/families/openfold3/tests/test_e2e.py index 3242dbe6a..4e9459f23 100644 --- a/families/openfold3/tests/test_e2e.py +++ b/families/openfold3/tests/test_e2e.py @@ -166,6 +166,16 @@ def _run_native( ) -> tuple[str, dict]: structure = output_root / f"prediction-{index}.cif" metadata = output_root / f"prediction-{index}.json" + # The qualification executable is built separately from the selected wheel. + # Resolve the staged core so copied and symlinked roots both select its + # companion loader instead of the executable's source-build RUNPATH. + library_root = (runtime_root / "libtrtmc_core.so").resolve(strict=True).parent + loader = library_root / "libtrtmc_runtime.so" + assert loader.is_file(), f"OpenFold3 runtime is missing its companion loader: {loader}" + environment = os.environ.copy() + environment["LD_LIBRARY_PATH"] = os.pathsep.join( + path for path in (str(library_root), environment.get("LD_LIBRARY_PATH", "")) if path + ) subprocess.run( [ str(qualification), @@ -177,6 +187,7 @@ def _run_native( ], check=True, timeout=timeout, + env=environment, ) return structure.read_text(encoding="utf-8"), json.loads(metadata.read_text(encoding="utf-8")) diff --git a/families/openfold3/tests/test_e2e_runtime.py b/families/openfold3/tests/test_e2e_runtime.py new file mode 100644 index 000000000..54148637d --- /dev/null +++ b/families/openfold3/tests/test_e2e_runtime.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from pathlib import Path +import shutil +import subprocess + +import pytest + +from . import test_e2e as e2e + + +@pytest.fixture(scope="module") +def qualification_builds(tmp_path_factory): + root = tmp_path_factory.mktemp("openfold3-builds") + repository = Path(__file__).resolve().parents[3] + sources = Path(__file__).with_name("cpp") + compiler = shutil.which("c++") + assert compiler, "OpenFold3 runtime regression requires a C++ compiler" + flags = [ + compiler, + "-std=c++17", + f"-I{repository / 'core/runtime/include'}", + f"-I{repository / 'families/openfold3/include'}", + ] + native, wheel = root / "native-build", root / "wheel" + for build_id, directory in enumerate((native, wheel), start=1): + directory.mkdir() + for name in ("core", "backend_trt"): + library = f"libtrtmc_{name}.so" + subprocess.run( + [ + *flags, + "-shared", + "-fPIC", + f"-DTEST_BUILD_ID={build_id}", + str(sources / "fake_build_identity.cpp"), + f"-Wl,-soname,{library}", + "-o", + str(directory / library), + ], + check=True, + ) + subprocess.run( + [ + *flags, + "-shared", + "-fPIC", + f"-DTEST_BUILD_ID={build_id}", + str(sources / "fake_qualification_runtime.cpp"), + f"-L{directory}", + "-ltrtmc_core", + "-ldl", + "-Wl,-soname,libtrtmc_runtime.so", + "-Wl,-rpath,$ORIGIN", + "-o", + str(directory / "libtrtmc_runtime.so"), + ], + check=True, + ) + qualification = native / "openfold3_qualification" + subprocess.run( + [ + *flags, + str(sources / "qualification.cpp"), + f"-L{native}", + "-ltrtmc_runtime", + f"-Wl,-rpath,{native}", + f"-Wl,-rpath-link,{native}", + "-o", + str(qualification), + ], + check=True, + ) + return qualification, wheel + + +@pytest.mark.parametrize("layout", ("copied", "symlinked")) +def test_qualification_uses_selected_product_build( + qualification_builds, monkeypatch, tmp_path: Path, layout: str +) -> None: + qualification, wheel = qualification_builds + runtime_root = tmp_path / "runtime" + runtime_root.mkdir() + for name in ("libtrtmc_core.so", "libtrtmc_backend_trt.so", "libtrtmc_runtime.so"): + if layout == "copied": + shutil.copy2(wheel / name, runtime_root / name) + elif name != "libtrtmc_runtime.so": + (runtime_root / name).symlink_to(wheel / name) + # The source-built executable's RUNPATH and the inherited environment both + # point at a different product build from the selected wheel libraries. + inherited_path = str(qualification.parent) + monkeypatch.setenv("LD_LIBRARY_PATH", inherited_path) + request = tmp_path / "query.json" + request.write_text("{}", encoding="utf-8") + + structure, metadata = e2e._run_native( + qualification, runtime_root, tmp_path / "model.bundle", request, tmp_path, 0, 30 + ) + + assert structure == "data_test\n" + assert metadata == {"build_id": 2} + assert os.environ["LD_LIBRARY_PATH"] == inherited_path + + +def test_qualification_rejects_a_missing_companion_loader(tmp_path: Path) -> None: + runtime_root = tmp_path / "runtime" + runtime_root.mkdir() + (runtime_root / "libtrtmc_core.so").touch() + + with pytest.raises(AssertionError, match="missing its companion loader"): + e2e._run_native( + tmp_path / "qualification", + runtime_root, + tmp_path / "model.bundle", + tmp_path / "query.json", + tmp_path, + 0, + 30, + )