Skip to content

poc version #4576

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
dtrawins wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

poc version #4576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
adf1fcf
poc version
dtrawins Sep 17, 2026
c7317a6
style
dtrawins Sep 17, 2026
9d2d26e
style
dtrawins Sep 17, 2026
abfbb6d
style1
dtrawins Sep 17, 2026
a848eb4
style2
dtrawins Sep 17, 2026
c272ef2
win-build-fix
dtrawins Sep 17, 2026
367d175
drop removed calculators
dtrawins Sep 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ci/lib_search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def check_header(fd):

def check_function(fd):
# Add space separated exceptions for given file in the dictionary
fix_applied = {"./src/test/ensemble_flow_custom_node_tests.cpp":"size_t strLen = std::strlen(str);size_t prefixLen = std::strlen(prefix);",}
fix_applied = {}

detected = False
try:
Expand Down
10 changes: 9 additions & 1 deletion install_redhat_gpu_drivers.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,15 @@ case $INSTALL_DRIVER_VERSION in \
rpm -ivh https://repositories.intel.com/gpu/rhel/9.6/pool/i/intel-level-zero-gpu-1.6.32567.19-1099.el9_5.x86_64.rpm ; \
rpm -ivh https://repositories.intel.com/gpu/rhel/9.6/pool/l/level-zero-1.20.2.0-1098.el9_5.x86_64.rpm ; \
;; \

"25.18.33578") \
$DNF_TOOL install --nodocs -y libedit libnl3; \
rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-gmmlib-22.7.2-i1146.el9_7.x86_64.rpm ; \
rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-igc-core-2.11.43-1146.el9_7.x86_64.rpm ; \
rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-igc-opencl-2.11.43-1146.el9_7.x86_64.rpm; \
rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-opencl-25.18.33578.77-1146.el9_7.x86_64.rpm ; \
rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-level-zero-gpu-1.6.33578.77-1146.el9_7.x86_64.rpm ; \
rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/l/level-zero-1.24.0.0-1146.el9_7.x86_64.rpm ; \
;; \
*) \
echo "ERROR: Unrecognized driver ${INSTALL_DRIVER_VERSION}." ; \
exit 1 ; \
Expand Down
66 changes: 1 addition & 65 deletions src/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ cc_shared_library(
"@com_github_libevent_libevent//:__subpackages__",
"@com_google_protobuf//:__subpackages__",
"@com_github_tencent_rapidjson//:__subpackages__",
"@org_tensorflow//:__subpackages__",
"@com_google_absl//:__subpackages__",
"@gif//:__subpackages__",
"@libjpeg_turbo//:__subpackages__",
Expand Down Expand Up @@ -135,14 +134,7 @@ ovms_cc_library(
"@mediapipe//mediapipe/framework/formats:image_frame",
"@mediapipe//mediapipe/framework/formats:image_frame_opencv",
"@mediapipe//mediapipe/framework/formats:tensor",
"@mediapipe//mediapipe/graphs/holistic_tracking:holistic_tracking_to_render_data",
"@mediapipe//mediapipe/graphs/iris_tracking:iris_tracking_cpu_deps",
"@mediapipe//mediapipe/calculators/tensor:image_to_tensor_calculator",
"@mediapipe//mediapipe/modules/holistic_landmark:holistic_landmark_cpu",
"@mediapipe//mediapipe/calculators/geti/inference:inference_calculators",
"@mediapipe//mediapipe/calculators/geti/utils:utils",
"@mediapipe//mediapipe/calculators/geti/utils:emptylabel_calculators",
"@mediapipe//mediapipe/calculators/geti/serialization:calculators",
"opencv_dep",
],
"//:disable_mediapipe": [],
Expand Down Expand Up @@ -609,7 +601,6 @@ ovms_cc_library(
"kfs_python_tensor_bridge",
"//src/mediapipe_internal:mediapipegraphexecutor_h",
"predict_request_validation_utils",
"tensorflow_type_utils",
"libovms_kfs_utils",
"libovms_kfs_grpc_inference_service_h",
"libovms_single_version_servable_definition",
Expand All @@ -627,33 +618,6 @@ ovms_cc_library(
visibility = ["//visibility:public",],
alwayslink = 1,
)
ovms_cc_library(
name = "kfs_graph_executor_impl_runtime",
srcs = [
"kfs_frontend/kfs_graph_executor_impl.cpp",
"kfs_frontend/kfs_graph_executor_impl.hpp",
],
deps = [
"kfs_python_tensor_bridge",
"//src/mediapipe_internal:mediapipegraphexecutor_h",
"predict_request_validation_utils",
"libovms_kfs_utils",
"libovms_kfs_grpc_inference_service_h",
"libovms_single_version_servable_definition",
"libovms_time_utils",
"//src/kfserving_api:kfserving_api_cpp",
"opencv_dep",
"@mediapipe//mediapipe/framework/formats:image_frame",
"@mediapipe//mediapipe/framework/formats:image_frame_opencv",
"@mediapipe//mediapipe/framework/formats:tensor",
],
additional_copts = select({
"//conditions:default": ["-fvisibility=default", "-DOVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME=1"],
"//src:windows": ["-DOVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME=1"],
}),
visibility = ["//visibility:public",],
alwayslink = 1,
)
ovms_cc_library(
name = "libovms_mediapipe_http_executor",
srcs = [
Expand Down Expand Up @@ -739,15 +703,6 @@ ovms_cc_library(
visibility = ["//visibility:public"],
)

ovms_cc_library(
name = "libovms_mediapipe_runtime_tensorflow_owner",
deps = [
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/lite/c:c_api",
],
visibility = ["//visibility:public"],
alwayslink = 1,
)
ovms_cc_library(
name = "customloaders",
hdrs = [
Expand Down Expand Up @@ -1245,19 +1200,6 @@ cc_import(
shared_library = "ovms_shared",
)

ovms_cc_library(

name = "tensorflow_type_utils",
hdrs = ["tensorflow_type_utils.hpp"],
srcs = ["tensorflow_type_utils.cpp"],
deps = [
"@org_tensorflow//tensorflow/core:framework",
"libovmsprecision",
"libovmsstatus",
],
visibility = ["//visibility:public"],
)

ovms_cc_library(
name = "libovmsschema",
hdrs = ["schema.hpp",],
Expand Down Expand Up @@ -2073,7 +2015,6 @@ cc_binary(
deps = [
"//src:ovms_lib",
"//src/filesystem:libovmsfilesystemfactory",
"//src:libovms_mediapipe_runtime_tensorflow_owner",
# NOTE: the MediaPipe runtime shared object is NOT linked into the main binary.
# Runtime loading keeps registration ownership outside of ovms and avoids
# duplicate symbol/registration conflicts.
Expand Down Expand Up @@ -2280,19 +2221,16 @@ cc_test(
"test/mediapipe/config_mediapipe_two_inputs.json",
"test/mediapipe/config_mediapipe_two_outputs_dag.json",
"test/mediapipe/config_mediapipe_multipart_mock.json",
"test/mediapipe/config_mp_tf_passthrough.json",
"test/mediapipe/config_standard_add.json",
"test/mediapipe/config_standard_dummy.json",
"test/mediapipe/graph_two_inputs_model.pbtxt",
"test/mediapipe/config_mediapipe_pytensor_ovtensor_converter.json",
"test/mediapipe/graph_gpt.pbtxt",
"test/mediapipe/graphadd.pbtxt",
"test/mediapipe/graphaddadapterfull.pbtxt",
"test/mediapipe/graphdummy_tf.pbtxt",
"test/mediapipe/graphdummy.pbtxt",
"test/mediapipe/graphdummyadapterfull.pbtxt",
"test/mediapipe/graphdummynonexistentcalculator.pbtxt",
"test/mediapipe/graphtfpassthrough.pbtxt",
"test/mediapipe/graphscalar.pbtxt",
"test/mediapipe/graphWithParams.pbtxt",
"test/mediapipe/graphdummyadapterfull_dummyinputnames.pbtxt",
Expand All @@ -2312,7 +2250,6 @@ cc_test(
"test/mediapipe/relative_paths/config_relative_add_subconfig_negative.json",
"test/mediapipe/relative_paths/config_relative_add_subconfig.json",
"test/mediapipe/relative_paths/config_relative_dummy.json",
"test/mediapipe/relative_paths/config_tflite_passthrough.json",
"test/mediapipe/relative_paths/config_relative_dummy_subconfig_base_path.json",
"test/mediapipe/relative_paths/graph1/dummy1/1/dummy.xml",
"test/mediapipe/relative_paths/graph1/graph.pbtxt",
Expand Down Expand Up @@ -2426,14 +2363,13 @@ cc_test(
":test_llm_input_processing_integration_tests",
"//src/test/mediapipe/calculators:mediapipe_test_calculators",
"//src/test/mediapipe/calculators:dependency_free_http_test_calculators",
"@mediapipe//mediapipe/calculators/ovms:ovms_calculator",
"//src/mediapipe_calculators/ovms:ovms_calculator",
"@mediapipe//mediapipe/framework:calculator_runner",
":text2image_test",
"//src/rerank:rerank_api_handler",
":embeddings_handler_tests",
":test_idle_mediapipe_test",
"//src/mediapipe_internal:mediapipe_utils",
"tensorflow_type_utils",
],
"//:disable_mediapipe" :
[
Expand Down
141 changes: 0 additions & 141 deletions src/kfs_frontend/kfs_graph_executor_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,6 @@
#include "../predict_request_validation_utils.hpp"
#include "../single_version_servable_definition.hpp"
#include "src/status.hpp"
#if !(defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME)
#include "../tensorflow_type_utils.hpp"
#endif
#include "src/kfs_python_tensor_bridge.hpp"

#pragma warning(push)
Expand Down Expand Up @@ -256,15 +253,13 @@ static Status receiveAndSerializePythonTensorIfSupported(
return StatusCode::OK;
}

#if defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME
static Status tfTensorRuntimeUnavailable(const std::string& streamName) {
std::stringstream ss;
ss << "TFTENSOR is not available in MediaPipe runtime KFS bridge for stream: " << streamName;
const std::string details = ss.str();
SPDLOG_DEBUG(details);
return Status(StatusCode::NOT_IMPLEMENTED, details);
}
#endif

static Status kfsPyTensorBridgeUnavailable(const std::string& streamName) {
std::stringstream ss;
Expand Down Expand Up @@ -431,28 +426,6 @@ static Status serializeKfsTypedContentToRawBytes(
return StatusCode::OK;
}

#if !(defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME)
template <>
Status receiveAndSerializePacket<tensorflow::Tensor>(const ::mediapipe::Packet& packet, KFSResponse& response, const std::string& outputStreamName) {
try {
auto& received = packet.Get<tensorflow::Tensor>();
auto* output = response.add_outputs();
output->set_name(outputStreamName);
output->set_datatype(
ovmsPrecisionToKFSPrecision(
TFSPrecisionToOvmsPrecision(
received.dtype())));
output->clear_shape();
for (const auto& dim : received.shape()) {
output->add_shape(dim.size);
}
response.add_raw_output_contents()->assign(reinterpret_cast<char*>(received.data()), received.TotalBytes());
return StatusCode::OK;
}
HANDLE_PACKET_RECEIVAL_EXCEPTIONS();
}
#endif

template <>
Status receiveAndSerializePacket<::mediapipe::Tensor>(const ::mediapipe::Packet& packet, KFSResponse& response, const std::string& outputStreamName) {
try {
Expand Down Expand Up @@ -678,112 +651,6 @@ static Status deserializeTensor(const std::string& requestedName, const KFSReque
return StatusCode::OK;
}

#if !(defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME)
static Status deserializeTensor(const std::string& requestedName, const KFSRequest& request, std::unique_ptr<tensorflow::Tensor>& outTensor, PythonBackend* pythonBackend) {
using tensorflow::Tensor;
using tensorflow::TensorShape;
auto requestInputItr = request.inputs().begin();
OVMS_RETURN_ON_FAIL(getRequestInput(requestInputItr, requestedName, request));
auto inputIndex = requestInputItr - request.inputs().begin();
try {
auto datatype = getPrecisionAsDataType(KFSPrecisionToOvmsPrecision(requestInputItr->datatype()));
if (datatype == TFSDataType::DT_INVALID) {
std::stringstream ss;
ss << "Not supported precision for Tensorflow tensor deserialization: " << requestInputItr->datatype();
const std::string details = ss.str();
SPDLOG_DEBUG(details);
return Status(StatusCode::INVALID_PRECISION, std::move(details));
}
TensorShape tensorShape;
std::vector<int64_t> rawShape;
for (int i = 0; i < requestInputItr->shape().size(); i++) {
if (requestInputItr->shape()[i] < 0) {
std::stringstream ss;
ss << "Negative dimension size is not acceptable: " << tensorShapeToString(requestInputItr->shape()) << "; input name: " << requestedName;
const std::string details = ss.str();
SPDLOG_DEBUG("[servable name: {} version: {}] Invalid shape - {}", request.model_name(), request.model_version(), details);
return Status(StatusCode::INVALID_SHAPE, details);
}
rawShape.emplace_back(requestInputItr->shape()[i]);
}
int64_t dimsCount = rawShape.size();
auto abslStatus = tensorflow::TensorShapeUtils::MakeShape(rawShape.data(), dimsCount, &tensorShape);
if (!abslStatus.ok()) {
auto stringViewAbslMessage = abslStatus.message();
return Status(StatusCode::UNKNOWN_ERROR, std::string{stringViewAbslMessage});
}
abslStatus = TensorShape::BuildTensorShapeBase(rawShape, static_cast<tensorflow::TensorShapeBase<TensorShape>*>(&tensorShape));
if (!abslStatus.ok()) {
auto stringViewAbslMessage = abslStatus.message();
return Status(StatusCode::UNKNOWN_ERROR, std::string{stringViewAbslMessage});
}
size_t expectedBytes = 1;
bool expectedBufferSizeValid = computeExpectedBufferSizeReturnFalseIfOverflow(rawShape, KFSDataTypeSize(requestInputItr->datatype()), expectedBytes);
if (!expectedBufferSizeValid) {
const std::string details = "Provided shape and datatype declare too large buffer.";
SPDLOG_DEBUG("[servable name: {} version: {}] {}", request.model_name(), request.model_version(), details);
return Status(StatusCode::INVALID_CONTENT_SIZE, details);
}
outTensor = std::make_unique<tensorflow::Tensor>(datatype, tensorShape);
if (request.raw_input_contents().size()) {
auto& bufferLocation = request.raw_input_contents().at(inputIndex);
if (outTensor->TotalBytes() != bufferLocation.size()) {
std::stringstream ss;
ss << "Mediapipe deserialization content size mismatch; allocated TF Tensor: " << outTensor->TotalBytes() << " bytes vs KServe buffer: " << bufferLocation.size() << " bytes";
const std::string details = ss.str();
SPDLOG_DEBUG("[servable name: {} version: {}] {}", request.model_name(), request.model_version(), details);
return Status(StatusCode::INVALID_CONTENT_SIZE, details);
}
void* tfTensordata = outTensor->data();
std::memcpy(tfTensordata, bufferLocation.data(), bufferLocation.size());
} else {
OVMS_RETURN_ON_FAIL(validateInputContent(*requestInputItr, expectedBytes, requestedName, request));
void* data = outTensor->data();
switch (datatype) {
case TFSDataType::DT_FLOAT: {
COPY_INPUT_VALUE_BY_VALUE(float, fp32);
}
case TFSDataType::DT_DOUBLE: {
COPY_INPUT_VALUE_BY_VALUE(double, fp64);
}
case TFSDataType::DT_INT64: {
COPY_INPUT_VALUE_BY_VALUE(int64_t, int64);
}
case TFSDataType::DT_INT32: {
COPY_INPUT_VALUE_BY_VALUE(int32_t, int);
}
case TFSDataType::DT_INT16: {
COPY_INPUT_VALUE_BY_VALUE(int16_t, int);
}
case TFSDataType::DT_INT8: {
COPY_INPUT_VALUE_BY_VALUE(int8_t, int);
}
case TFSDataType::DT_UINT64: {
COPY_INPUT_VALUE_BY_VALUE(uint64_t, uint64);
}
case TFSDataType::DT_UINT32: {
COPY_INPUT_VALUE_BY_VALUE(uint32_t, uint);
}
case TFSDataType::DT_UINT16: {
COPY_INPUT_VALUE_BY_VALUE(uint16_t, uint);
}
case TFSDataType::DT_UINT8: {
COPY_INPUT_VALUE_BY_VALUE(uint8_t, uint);
}
case TFSDataType::DT_BOOL: {
COPY_INPUT_VALUE_BY_VALUE(bool, bool);
}
case TFSDataType::DT_HALF:
default:
return ovms::Status(ovms::StatusCode::NOT_IMPLEMENTED, "There is no support for types different than fp32, int64, int32, uint32, uint64, int8, uint8, bool");
}
}
}
HANDLE_DESERIALIZATION_EXCEPTION("Tensorflow tensor")
return StatusCode::OK;
}
#endif

static Status deserializeTensor(const std::string& requestedName, const KFSRequest& request, std::unique_ptr<ov::Tensor>& outTensor, PythonBackend* pythonBackend) {
auto requestInputItr = request.inputs().begin();
OVMS_RETURN_ON_FAIL(getRequestInput(requestInputItr, requestedName, request));
Expand Down Expand Up @@ -1171,11 +1038,7 @@ static Status createPacketAndPushIntoGraph(const std::string& inputName, std::sh
status = createPacketAndPushIntoGraph<Holder>(inputName, request, graph, timestamp, nullptr);
} else if (inputPacketType == mediapipe_packet_type_enum::TFTENSOR) {
SPDLOG_DEBUG("Request processing TF tensor: {}", inputName);
#if defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME
status = tfTensorRuntimeUnavailable(inputName);
#else
status = createPacketAndPushIntoGraph<tensorflow::Tensor, Holder>(inputName, request, graph, timestamp, nullptr);
#endif
} else if (inputPacketType == mediapipe_packet_type_enum::MPTENSOR) {
SPDLOG_DEBUG("Request processing MP tensor: {}", inputName);
status = createPacketAndPushIntoGraph<mediapipe::Tensor, Holder>(inputName, request, graph, timestamp, nullptr);
Expand Down Expand Up @@ -1270,11 +1133,7 @@ Status onPacketReadySerializeImpl(
status = receiveAndSerializePacket<KFSResponse>(packet, response, packetName);
} else if (packetType == mediapipe_packet_type_enum::TFTENSOR) {
SPDLOG_DEBUG("Response processing packet type TF Tensor name: {}", packetName);
#if defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME
status = tfTensorRuntimeUnavailable(packetName);
#else
status = receiveAndSerializePacket<tensorflow::Tensor>(packet, response, packetName);
#endif
} else if (packetType == mediapipe_packet_type_enum::TFLITETENSOR) {
SPDLOG_DEBUG("Response processing packet type TFLite Tensor name: {}", packetName);
std::string details{"Response processing packet type TFLite Tensor is not supported"};
Expand Down
Loading