From adf1fcf047502309648b6671b0a8c48ffcaf530a Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 17 Sep 2026 07:51:50 +0200 Subject: [PATCH 01/10] poc version --- ci/lib_search.py | 2 +- install_redhat_gpu_drivers.sh | 10 +- src/BUILD | 66 +- src/kfs_frontend/kfs_graph_executor_impl.cpp | 141 - src/mediapipe_calculators/ovms/BUILD | 135 + .../ovms/modelapiovmsadapter.cc | 404 ++ .../ovms/modelapiovmsadapter.hpp | 84 + .../ovms/openvinoinferencecalculator.cc | 390 + .../ovms/openvinoinferencecalculator.h | 42 + .../ovms/openvinoinferencecalculator.proto | 33 + .../openvinoinferencecalculatoroptions.cc | 204 + .../ovms/openvinoinferencecalculatoroptions.h | 42 + .../ovms/openvinoinferencedumputils.cc | 148 + .../ovms/openvinoinferencedumputils.h | 28 + .../ovms/openvinoinferenceutils.cc | 91 + .../ovms/openvinoinferenceutils.h} | 32 +- .../openvinomodelserversessioncalculator.cc | 209 + .../openvinomodelserversessioncalculator.h | 52 + ...openvinomodelserversessioncalculator.proto | 35 + src/mediapipe_internal/BUILD | 6 +- .../mediapipegraphdefinition.cpp | 5 - src/tensorflow_type_utils.cpp | 67 - src/test/ensemble_flow_custom_node_tests.cpp | 5976 --------------- src/test/ensemble_tests.cpp | 6376 ----------------- src/test/gather_node_test.cpp | 376 - .../mediapipe/config_mp_tf_passthrough.json | 31 - src/test/mediapipe/graphdummy_tf.pbtxt | 45 - src/test/mediapipe/graphdummy_tflite.pbtxt | 45 - src/test/mediapipe/graphtfpassthrough.pbtxt | 22 - .../config_tflite_passthrough.json | 11 - src/test/mediapipe_framework_test.cpp | 1 - src/test/mediapipeflow_test.cpp | 218 +- src/test/pythonnode_test.cpp | 1 - src/test/stress_test_utils.hpp | 1 - third_party/mediapipe_calculators/BUILD | 6 +- third_party/tf_text/BUILD | 17 - third_party/tf_text/tftext.patch | 38 - 37 files changed, 1927 insertions(+), 13463 deletions(-) create mode 100644 src/mediapipe_calculators/ovms/BUILD create mode 100644 src/mediapipe_calculators/ovms/modelapiovmsadapter.cc create mode 100644 src/mediapipe_calculators/ovms/modelapiovmsadapter.hpp create mode 100644 src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc create mode 100644 src/mediapipe_calculators/ovms/openvinoinferencecalculator.h create mode 100644 src/mediapipe_calculators/ovms/openvinoinferencecalculator.proto create mode 100644 src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc create mode 100644 src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h create mode 100644 src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc create mode 100644 src/mediapipe_calculators/ovms/openvinoinferencedumputils.h create mode 100644 src/mediapipe_calculators/ovms/openvinoinferenceutils.cc rename src/{tensorflow_type_utils.hpp => mediapipe_calculators/ovms/openvinoinferenceutils.h} (51%) create mode 100644 src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.cc create mode 100644 src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.h create mode 100644 src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.proto delete mode 100644 src/tensorflow_type_utils.cpp delete mode 100644 src/test/ensemble_flow_custom_node_tests.cpp delete mode 100644 src/test/ensemble_tests.cpp delete mode 100644 src/test/gather_node_test.cpp delete mode 100644 src/test/mediapipe/config_mp_tf_passthrough.json delete mode 100644 src/test/mediapipe/graphdummy_tf.pbtxt delete mode 100644 src/test/mediapipe/graphdummy_tflite.pbtxt delete mode 100644 src/test/mediapipe/graphtfpassthrough.pbtxt delete mode 100644 src/test/mediapipe/relative_paths/config_tflite_passthrough.json delete mode 100644 third_party/tf_text/BUILD delete mode 100644 third_party/tf_text/tftext.patch diff --git a/ci/lib_search.py b/ci/lib_search.py index e287adfda8..af07283335 100644 --- a/ci/lib_search.py +++ b/ci/lib_search.py @@ -34,7 +34,7 @@ def check_header(fd): def check_function(fd): # Add space separated exceptions for given file in the dictionary - fix_applied = {"./src/test/ensemble_flow_custom_node_tests.cpp":"size_t strLen = std::strlen(str);size_t prefixLen = std::strlen(prefix);",} + fix_applied = {} detected = False try: diff --git a/install_redhat_gpu_drivers.sh b/install_redhat_gpu_drivers.sh index 6f2c9f4902..6ad40434c7 100755 --- a/install_redhat_gpu_drivers.sh +++ b/install_redhat_gpu_drivers.sh @@ -54,7 +54,15 @@ case $INSTALL_DRIVER_VERSION in \ rpm -ivh https://repositories.intel.com/gpu/rhel/9.6/pool/i/intel-level-zero-gpu-1.6.32567.19-1099.el9_5.x86_64.rpm ; \ rpm -ivh https://repositories.intel.com/gpu/rhel/9.6/pool/l/level-zero-1.20.2.0-1098.el9_5.x86_64.rpm ; \ ;; \ - +"25.18.33578") \ + $DNF_TOOL install --nodocs -y libedit libnl3; \ + rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-gmmlib-22.7.2-i1146.el9_7.x86_64.rpm ; \ + rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-igc-core-2.11.43-1146.el9_7.x86_64.rpm ; \ + rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-igc-opencl-2.11.43-1146.el9_7.x86_64.rpm; \ + rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-opencl-25.18.33578.77-1146.el9_7.x86_64.rpm ; \ + rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/i/intel-level-zero-gpu-1.6.33578.77-1146.el9_7.x86_64.rpm ; \ + rpm -ivh https://repositories.intel.com/gpu/rhel/9.7/pool/l/level-zero-1.24.0.0-1146.el9_7.x86_64.rpm ; \ +;; \ *) \ echo "ERROR: Unrecognized driver ${INSTALL_DRIVER_VERSION}." ; \ exit 1 ; \ diff --git a/src/BUILD b/src/BUILD index 744139727d..0e723d0249 100644 --- a/src/BUILD +++ b/src/BUILD @@ -62,7 +62,6 @@ cc_shared_library( "@com_github_libevent_libevent//:__subpackages__", "@com_google_protobuf//:__subpackages__", "@com_github_tencent_rapidjson//:__subpackages__", - "@org_tensorflow//:__subpackages__", "@com_google_absl//:__subpackages__", "@gif//:__subpackages__", "@libjpeg_turbo//:__subpackages__", @@ -135,14 +134,7 @@ ovms_cc_library( "@mediapipe//mediapipe/framework/formats:image_frame", "@mediapipe//mediapipe/framework/formats:image_frame_opencv", "@mediapipe//mediapipe/framework/formats:tensor", - "@mediapipe//mediapipe/graphs/holistic_tracking:holistic_tracking_to_render_data", - "@mediapipe//mediapipe/graphs/iris_tracking:iris_tracking_cpu_deps", "@mediapipe//mediapipe/calculators/tensor:image_to_tensor_calculator", - "@mediapipe//mediapipe/modules/holistic_landmark:holistic_landmark_cpu", - "@mediapipe//mediapipe/calculators/geti/inference:inference_calculators", - "@mediapipe//mediapipe/calculators/geti/utils:utils", - "@mediapipe//mediapipe/calculators/geti/utils:emptylabel_calculators", - "@mediapipe//mediapipe/calculators/geti/serialization:calculators", "opencv_dep", ], "//:disable_mediapipe": [], @@ -609,7 +601,6 @@ ovms_cc_library( "kfs_python_tensor_bridge", "//src/mediapipe_internal:mediapipegraphexecutor_h", "predict_request_validation_utils", - "tensorflow_type_utils", "libovms_kfs_utils", "libovms_kfs_grpc_inference_service_h", "libovms_single_version_servable_definition", @@ -627,33 +618,6 @@ ovms_cc_library( visibility = ["//visibility:public",], alwayslink = 1, ) -ovms_cc_library( - name = "kfs_graph_executor_impl_runtime", - srcs = [ - "kfs_frontend/kfs_graph_executor_impl.cpp", - "kfs_frontend/kfs_graph_executor_impl.hpp", - ], - deps = [ - "kfs_python_tensor_bridge", - "//src/mediapipe_internal:mediapipegraphexecutor_h", - "predict_request_validation_utils", - "libovms_kfs_utils", - "libovms_kfs_grpc_inference_service_h", - "libovms_single_version_servable_definition", - "libovms_time_utils", - "//src/kfserving_api:kfserving_api_cpp", - "opencv_dep", - "@mediapipe//mediapipe/framework/formats:image_frame", - "@mediapipe//mediapipe/framework/formats:image_frame_opencv", - "@mediapipe//mediapipe/framework/formats:tensor", - ], - additional_copts = select({ - "//conditions:default": ["-fvisibility=default", "-DOVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME=1"], - "//src:windows": ["-DOVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME=1"], - }), - visibility = ["//visibility:public",], - alwayslink = 1, -) ovms_cc_library( name = "libovms_mediapipe_http_executor", srcs = [ @@ -739,15 +703,6 @@ ovms_cc_library( visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "libovms_mediapipe_runtime_tensorflow_owner", - deps = [ - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/lite/c:c_api", - ], - visibility = ["//visibility:public"], - alwayslink = 1, -) ovms_cc_library( name = "customloaders", hdrs = [ @@ -1245,19 +1200,6 @@ cc_import( shared_library = "ovms_shared", ) -ovms_cc_library( - - name = "tensorflow_type_utils", - hdrs = ["tensorflow_type_utils.hpp"], - srcs = ["tensorflow_type_utils.cpp"], - deps = [ - "@org_tensorflow//tensorflow/core:framework", - "libovmsprecision", - "libovmsstatus", - ], - visibility = ["//visibility:public"], -) - ovms_cc_library( name = "libovmsschema", hdrs = ["schema.hpp",], @@ -2073,7 +2015,6 @@ cc_binary( deps = [ "//src:ovms_lib", "//src/filesystem:libovmsfilesystemfactory", - "//src:libovms_mediapipe_runtime_tensorflow_owner", # NOTE: the MediaPipe runtime shared object is NOT linked into the main binary. # Runtime loading keeps registration ownership outside of ovms and avoids # duplicate symbol/registration conflicts. @@ -2280,7 +2221,6 @@ cc_test( "test/mediapipe/config_mediapipe_two_inputs.json", "test/mediapipe/config_mediapipe_two_outputs_dag.json", "test/mediapipe/config_mediapipe_multipart_mock.json", - "test/mediapipe/config_mp_tf_passthrough.json", "test/mediapipe/config_standard_add.json", "test/mediapipe/config_standard_dummy.json", "test/mediapipe/graph_two_inputs_model.pbtxt", @@ -2288,11 +2228,9 @@ cc_test( "test/mediapipe/graph_gpt.pbtxt", "test/mediapipe/graphadd.pbtxt", "test/mediapipe/graphaddadapterfull.pbtxt", - "test/mediapipe/graphdummy_tf.pbtxt", "test/mediapipe/graphdummy.pbtxt", "test/mediapipe/graphdummyadapterfull.pbtxt", "test/mediapipe/graphdummynonexistentcalculator.pbtxt", - "test/mediapipe/graphtfpassthrough.pbtxt", "test/mediapipe/graphscalar.pbtxt", "test/mediapipe/graphWithParams.pbtxt", "test/mediapipe/graphdummyadapterfull_dummyinputnames.pbtxt", @@ -2312,7 +2250,6 @@ cc_test( "test/mediapipe/relative_paths/config_relative_add_subconfig_negative.json", "test/mediapipe/relative_paths/config_relative_add_subconfig.json", "test/mediapipe/relative_paths/config_relative_dummy.json", - "test/mediapipe/relative_paths/config_tflite_passthrough.json", "test/mediapipe/relative_paths/config_relative_dummy_subconfig_base_path.json", "test/mediapipe/relative_paths/graph1/dummy1/1/dummy.xml", "test/mediapipe/relative_paths/graph1/graph.pbtxt", @@ -2426,14 +2363,13 @@ cc_test( ":test_llm_input_processing_integration_tests", "//src/test/mediapipe/calculators:mediapipe_test_calculators", "//src/test/mediapipe/calculators:dependency_free_http_test_calculators", - "@mediapipe//mediapipe/calculators/ovms:ovms_calculator", + "//src/mediapipe_calculators/ovms:ovms_calculator", "@mediapipe//mediapipe/framework:calculator_runner", ":text2image_test", "//src/rerank:rerank_api_handler", ":embeddings_handler_tests", ":test_idle_mediapipe_test", "//src/mediapipe_internal:mediapipe_utils", - "tensorflow_type_utils", ], "//:disable_mediapipe" : [ diff --git a/src/kfs_frontend/kfs_graph_executor_impl.cpp b/src/kfs_frontend/kfs_graph_executor_impl.cpp index da450a06a3..f29272b3dc 100644 --- a/src/kfs_frontend/kfs_graph_executor_impl.cpp +++ b/src/kfs_frontend/kfs_graph_executor_impl.cpp @@ -31,9 +31,6 @@ #include "../predict_request_validation_utils.hpp" #include "../single_version_servable_definition.hpp" #include "src/status.hpp" -#if !(defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) -#include "../tensorflow_type_utils.hpp" -#endif #include "src/kfs_python_tensor_bridge.hpp" #pragma warning(push) @@ -256,7 +253,6 @@ static Status receiveAndSerializePythonTensorIfSupported( return StatusCode::OK; } -#if defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME static Status tfTensorRuntimeUnavailable(const std::string& streamName) { std::stringstream ss; ss << "TFTENSOR is not available in MediaPipe runtime KFS bridge for stream: " << streamName; @@ -264,7 +260,6 @@ static Status tfTensorRuntimeUnavailable(const std::string& streamName) { SPDLOG_DEBUG(details); return Status(StatusCode::NOT_IMPLEMENTED, details); } -#endif static Status kfsPyTensorBridgeUnavailable(const std::string& streamName) { std::stringstream ss; @@ -431,28 +426,6 @@ static Status serializeKfsTypedContentToRawBytes( return StatusCode::OK; } -#if !(defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) -template <> -Status receiveAndSerializePacket(const ::mediapipe::Packet& packet, KFSResponse& response, const std::string& outputStreamName) { - try { - auto& received = packet.Get(); - auto* output = response.add_outputs(); - output->set_name(outputStreamName); - output->set_datatype( - ovmsPrecisionToKFSPrecision( - TFSPrecisionToOvmsPrecision( - received.dtype()))); - output->clear_shape(); - for (const auto& dim : received.shape()) { - output->add_shape(dim.size); - } - response.add_raw_output_contents()->assign(reinterpret_cast(received.data()), received.TotalBytes()); - return StatusCode::OK; - } - HANDLE_PACKET_RECEIVAL_EXCEPTIONS(); -} -#endif - template <> Status receiveAndSerializePacket<::mediapipe::Tensor>(const ::mediapipe::Packet& packet, KFSResponse& response, const std::string& outputStreamName) { try { @@ -678,112 +651,6 @@ static Status deserializeTensor(const std::string& requestedName, const KFSReque return StatusCode::OK; } -#if !(defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) -static Status deserializeTensor(const std::string& requestedName, const KFSRequest& request, std::unique_ptr& outTensor, PythonBackend* pythonBackend) { - using tensorflow::Tensor; - using tensorflow::TensorShape; - auto requestInputItr = request.inputs().begin(); - OVMS_RETURN_ON_FAIL(getRequestInput(requestInputItr, requestedName, request)); - auto inputIndex = requestInputItr - request.inputs().begin(); - try { - auto datatype = getPrecisionAsDataType(KFSPrecisionToOvmsPrecision(requestInputItr->datatype())); - if (datatype == TFSDataType::DT_INVALID) { - std::stringstream ss; - ss << "Not supported precision for Tensorflow tensor deserialization: " << requestInputItr->datatype(); - const std::string details = ss.str(); - SPDLOG_DEBUG(details); - return Status(StatusCode::INVALID_PRECISION, std::move(details)); - } - TensorShape tensorShape; - std::vector rawShape; - for (int i = 0; i < requestInputItr->shape().size(); i++) { - if (requestInputItr->shape()[i] < 0) { - std::stringstream ss; - ss << "Negative dimension size is not acceptable: " << tensorShapeToString(requestInputItr->shape()) << "; input name: " << requestedName; - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid shape - {}", request.model_name(), request.model_version(), details); - return Status(StatusCode::INVALID_SHAPE, details); - } - rawShape.emplace_back(requestInputItr->shape()[i]); - } - int64_t dimsCount = rawShape.size(); - auto abslStatus = tensorflow::TensorShapeUtils::MakeShape(rawShape.data(), dimsCount, &tensorShape); - if (!abslStatus.ok()) { - auto stringViewAbslMessage = abslStatus.message(); - return Status(StatusCode::UNKNOWN_ERROR, std::string{stringViewAbslMessage}); - } - abslStatus = TensorShape::BuildTensorShapeBase(rawShape, static_cast*>(&tensorShape)); - if (!abslStatus.ok()) { - auto stringViewAbslMessage = abslStatus.message(); - return Status(StatusCode::UNKNOWN_ERROR, std::string{stringViewAbslMessage}); - } - size_t expectedBytes = 1; - bool expectedBufferSizeValid = computeExpectedBufferSizeReturnFalseIfOverflow(rawShape, KFSDataTypeSize(requestInputItr->datatype()), expectedBytes); - if (!expectedBufferSizeValid) { - const std::string details = "Provided shape and datatype declare too large buffer."; - SPDLOG_DEBUG("[servable name: {} version: {}] {}", request.model_name(), request.model_version(), details); - return Status(StatusCode::INVALID_CONTENT_SIZE, details); - } - outTensor = std::make_unique(datatype, tensorShape); - if (request.raw_input_contents().size()) { - auto& bufferLocation = request.raw_input_contents().at(inputIndex); - if (outTensor->TotalBytes() != bufferLocation.size()) { - std::stringstream ss; - ss << "Mediapipe deserialization content size mismatch; allocated TF Tensor: " << outTensor->TotalBytes() << " bytes vs KServe buffer: " << bufferLocation.size() << " bytes"; - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] {}", request.model_name(), request.model_version(), details); - return Status(StatusCode::INVALID_CONTENT_SIZE, details); - } - void* tfTensordata = outTensor->data(); - std::memcpy(tfTensordata, bufferLocation.data(), bufferLocation.size()); - } else { - OVMS_RETURN_ON_FAIL(validateInputContent(*requestInputItr, expectedBytes, requestedName, request)); - void* data = outTensor->data(); - switch (datatype) { - case TFSDataType::DT_FLOAT: { - COPY_INPUT_VALUE_BY_VALUE(float, fp32); - } - case TFSDataType::DT_DOUBLE: { - COPY_INPUT_VALUE_BY_VALUE(double, fp64); - } - case TFSDataType::DT_INT64: { - COPY_INPUT_VALUE_BY_VALUE(int64_t, int64); - } - case TFSDataType::DT_INT32: { - COPY_INPUT_VALUE_BY_VALUE(int32_t, int); - } - case TFSDataType::DT_INT16: { - COPY_INPUT_VALUE_BY_VALUE(int16_t, int); - } - case TFSDataType::DT_INT8: { - COPY_INPUT_VALUE_BY_VALUE(int8_t, int); - } - case TFSDataType::DT_UINT64: { - COPY_INPUT_VALUE_BY_VALUE(uint64_t, uint64); - } - case TFSDataType::DT_UINT32: { - COPY_INPUT_VALUE_BY_VALUE(uint32_t, uint); - } - case TFSDataType::DT_UINT16: { - COPY_INPUT_VALUE_BY_VALUE(uint16_t, uint); - } - case TFSDataType::DT_UINT8: { - COPY_INPUT_VALUE_BY_VALUE(uint8_t, uint); - } - case TFSDataType::DT_BOOL: { - COPY_INPUT_VALUE_BY_VALUE(bool, bool); - } - case TFSDataType::DT_HALF: - default: - return ovms::Status(ovms::StatusCode::NOT_IMPLEMENTED, "There is no support for types different than fp32, int64, int32, uint32, uint64, int8, uint8, bool"); - } - } - } - HANDLE_DESERIALIZATION_EXCEPTION("Tensorflow tensor") - return StatusCode::OK; -} -#endif - static Status deserializeTensor(const std::string& requestedName, const KFSRequest& request, std::unique_ptr& outTensor, PythonBackend* pythonBackend) { auto requestInputItr = request.inputs().begin(); OVMS_RETURN_ON_FAIL(getRequestInput(requestInputItr, requestedName, request)); @@ -1171,11 +1038,7 @@ static Status createPacketAndPushIntoGraph(const std::string& inputName, std::sh status = createPacketAndPushIntoGraph(inputName, request, graph, timestamp, nullptr); } else if (inputPacketType == mediapipe_packet_type_enum::TFTENSOR) { SPDLOG_DEBUG("Request processing TF tensor: {}", inputName); -#if defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME status = tfTensorRuntimeUnavailable(inputName); -#else - status = createPacketAndPushIntoGraph(inputName, request, graph, timestamp, nullptr); -#endif } else if (inputPacketType == mediapipe_packet_type_enum::MPTENSOR) { SPDLOG_DEBUG("Request processing MP tensor: {}", inputName); status = createPacketAndPushIntoGraph(inputName, request, graph, timestamp, nullptr); @@ -1270,11 +1133,7 @@ Status onPacketReadySerializeImpl( status = receiveAndSerializePacket(packet, response, packetName); } else if (packetType == mediapipe_packet_type_enum::TFTENSOR) { SPDLOG_DEBUG("Response processing packet type TF Tensor name: {}", packetName); -#if defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME status = tfTensorRuntimeUnavailable(packetName); -#else - status = receiveAndSerializePacket(packet, response, packetName); -#endif } else if (packetType == mediapipe_packet_type_enum::TFLITETENSOR) { SPDLOG_DEBUG("Response processing packet type TFLite Tensor name: {}", packetName); std::string details{"Response processing packet type TFLite Tensor is not supported"}; diff --git a/src/mediapipe_calculators/ovms/BUILD b/src/mediapipe_calculators/ovms/BUILD new file mode 100644 index 0000000000..a3acc6dd86 --- /dev/null +++ b/src/mediapipe_calculators/ovms/BUILD @@ -0,0 +1,135 @@ +# +# Copyright (c) 2023-2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +mediapipe_proto_library( + name = "openvinoinferencecalculator_proto", + srcs = ["openvinoinferencecalculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "@mediapipe//mediapipe/framework:calculator_options_proto", + "@mediapipe//mediapipe/framework:calculator_proto", + ], +) + +mediapipe_proto_library( + name = "openvinomodelserversessioncalculator_proto", + srcs = ["openvinomodelserversessioncalculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "@mediapipe//mediapipe/framework:calculator_options_proto", + "@mediapipe//mediapipe/framework:calculator_proto", + ], +) + +cc_library( + name = "openvinoinferencedumputils", + srcs = ["openvinoinferencedumputils.cc"], + hdrs = ["openvinoinferencedumputils.h"], + copts = ["-Isrc"], + deps = [ + "//third_party:openvino", + ], + alwayslink = 1, +) + +cc_library( + name = "openvinoinferenceutils", + srcs = ["openvinoinferenceutils.cc"], + hdrs = ["openvinoinferenceutils.h"], + copts = ["-Isrc"], + deps = [ + "//src:ovms_header", + ], + alwayslink = 1, +) + +cc_library( + name = "modelapiovmsadapter", + srcs = ["modelapiovmsadapter.cc"], + hdrs = ["modelapiovmsadapter.hpp"], + copts = ["-Isrc"], + deps = [ + ":openvinoinferencedumputils", + "//src:ovms_header", + "//third_party:openvino", + "@mediapipe//mediapipe/framework/port:logging", + "@model_api//:model_api", + ], +) + +cc_library( + name = "openvinoinferencecalculatoroptions", + srcs = ["openvinoinferencecalculatoroptions.cc"], + hdrs = ["openvinoinferencecalculatoroptions.h"], + copts = ["-Isrc"], + deps = [ + ":openvinoinferencecalculator_cc_proto", + ":openvinoinferenceutils", + "//src:ovms_header", + "@mediapipe//mediapipe/framework:calculator_framework", + ], + alwayslink = 1, +) + +cc_library( + name = "openvinoinferencecalculator", + srcs = ["openvinoinferencecalculator.cc"], + hdrs = ["openvinoinferencecalculator.h"], + copts = ["-Isrc"], + deps = [ + ":modelapiovmsadapter", + ":openvinoinferencecalculator_cc_proto", + ":openvinoinferencecalculatoroptions", + ":openvinoinferenceutils", + "//src:ovms_header", + "//third_party:openvino", + "@mediapipe//mediapipe/framework:calculator_framework", + "@mediapipe//mediapipe/framework/formats:tensor", + ], + alwayslink = 1, +) + +cc_library( + name = "openvinomodelserversessioncalculator", + srcs = ["openvinomodelserversessioncalculator.cc"], + hdrs = ["openvinomodelserversessioncalculator.h"], + copts = ["-Isrc"], + deps = [ + ":modelapiovmsadapter", + ":openvinoinferenceutils", + ":openvinomodelserversessioncalculator_cc_proto", + "//src:ovms_header", + "//third_party:openvino", + "@mediapipe//mediapipe/framework:calculator_framework", + ], + alwayslink = 1, +) + +cc_library( + name = "ovms_calculator", + deps = [ + ":modelapiovmsadapter", + ":openvinoinferencecalculator", + ":openvinomodelserversessioncalculator", + ], + alwayslink = 1, +) diff --git a/src/mediapipe_calculators/ovms/modelapiovmsadapter.cc b/src/mediapipe_calculators/ovms/modelapiovmsadapter.cc new file mode 100644 index 0000000000..76aafe3476 --- /dev/null +++ b/src/mediapipe_calculators/ovms/modelapiovmsadapter.cc @@ -0,0 +1,404 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "modelapiovmsadapter.hpp" +#if (OVMS_DUMP_TO_FILE == 1) +#include "openvinoinferencedumputils.h" +#endif +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ovms.h" // NOLINT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/port/logging.h" +#pragma GCC diagnostic pop +namespace mediapipe::ovms { + +using std::endl; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +using InferenceOutput = std::map; +using InferenceInput = std::map; + +#define THROW_IF_CIRCULAR_ERR(C_API_CALL) \ + { \ + auto* fatalErr = C_API_CALL; \ + if (fatalErr != nullptr) { \ + std::runtime_error exc("Getting status details circular error"); \ + throw exc; \ + } \ + } + +#define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ + { \ + auto* err = C_API_CALL; \ + if (err != nullptr) { \ + uint32_t code = 0; \ + const char* msg = nullptr; \ + THROW_IF_CIRCULAR_ERR(OVMS_StatusCode(err, &code)); \ + THROW_IF_CIRCULAR_ERR(OVMS_StatusDetails(err, &msg)); \ + LOG(INFO) << "Error encountred in OVMSCalculator:" << msg << " code: " << code; \ + std::runtime_error exc(msg); \ + OVMS_StatusDelete(err); \ + throw exc; \ + } \ + } + +#define CREATE_GUARD(GUARD_NAME, CAPI_TYPE, CAPI_PTR) \ + std::unique_ptr GUARD_NAME(CAPI_PTR, &(CAPI_TYPE##Delete)); + +static OVMS_DataType OVPrecision2CAPI(ov::element::Type_t datatype); +static ov::element::Type_t CAPI2OVPrecision(OVMS_DataType datatype); +static ov::Tensor makeOvTensor(OVMS_DataType datatype, const int64_t* shape, size_t dimCount, const void* voutputData, size_t bytesize); + +OVMSInferenceAdapter::~OVMSInferenceAdapter() { + LOG(INFO) << "OVMSAdapter destr"; +} + +inline std::vector getShapeAcceptableByCAPI(const ov::Shape& shape) { + if (std::any_of(shape.begin(), shape.end(), [](size_t dim) { return dim > std::numeric_limits::max(); })) { + throw std::runtime_error("Cannot use C-API with dimension size greater than int64_t max value"); + } + return std::vector{shape.begin(), shape.end()}; +} + +void OVMSInferenceAdapter::infer(const InferenceInput& input, InferenceOutput& output) { + OVMS_InferenceRequest* request{nullptr}; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, servableName.c_str(), servableVersion)); + CREATE_GUARD(requestGuard, OVMS_InferenceRequest, request); + + OVMS_Status* status{nullptr}; + std::vector outputsSet; + for (const auto& [name, input_tensor] : input) { + const char* realName = name.c_str(); + const auto& ovShape = input_tensor.get_shape(); + std::vector capiShape = getShapeAcceptableByCAPI(ovShape); + OVMS_DataType inputDataType = OVPrecision2CAPI(input_tensor.get_element_type()); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realName, inputDataType, capiShape.data(), capiShape.size())); + const uint32_t NOT_USED_NUM = 0; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, + realName, + reinterpret_cast(input_tensor.data()), + input_tensor.get_byte_size(), + OVMS_BUFFERTYPE_CPU, + NOT_USED_NUM)); + } + for (const auto& [name, output_tensor] : output) { + outputsSet.emplace_back(name); + const char* realName = name.c_str(); + const auto& ovShape = output_tensor.get_shape(); + std::vector capiShape = getShapeAcceptableByCAPI(ovShape); + OVMS_DataType inputDataType = OVPrecision2CAPI(output_tensor.get_element_type()); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddOutput(request, realName, inputDataType, capiShape.data(), capiShape.size())); + const uint32_t NOT_USED_NUM = 0; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestOutputSetData(request, + realName, + reinterpret_cast(output_tensor.data()), + output_tensor.get_byte_size(), + OVMS_BUFFERTYPE_CPU, + NOT_USED_NUM)); + } +#if (OVMS_DUMP_TO_FILE == 1) + dumpOvTensorInput(input, "input"); +#endif + OVMS_InferenceResponse* response = nullptr; + ASSERT_CAPI_STATUS_NULL(OVMS_Inference(cserver, request, &response)); + CREATE_GUARD(responseGuard, OVMS_InferenceResponse, response); + uint32_t outputCount = 42; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutputCount(response, &outputCount)); + uint32_t parameterCount = 42; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseParameterCount(response, ¶meterCount)); + const void* voutputData; + size_t bytesize = 42; + OVMS_DataType datatype = (OVMS_DataType)199; + const int64_t* shape{nullptr}; + size_t dimCount = 42; + OVMS_BufferType bufferType = (OVMS_BufferType)199; + uint32_t deviceId = 42; + const char* outputName{nullptr}; + for (size_t i = 0; i < outputCount; ++i) { + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutput(response, i, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); + if (std::find(outputsSet.begin(), outputsSet.end(), outputName) == outputsSet.end()) { + output.emplace(outputName, std::move(makeOvTensor(datatype, shape, dimCount, voutputData, bytesize))); + } + } +#if (OVMS_DUMP_TO_FILE == 1) + dumpOvTensorInput(output, "output"); +#endif +} + +InferenceOutput OVMSInferenceAdapter::infer(const InferenceInput& input) { + OVMS_InferenceRequest* request{nullptr}; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, servableName.c_str(), servableVersion)); + CREATE_GUARD(requestGuard, OVMS_InferenceRequest, request); + + InferenceOutput output; + OVMS_Status* status{nullptr}; + for (const auto& [name, input_tensor] : input) { + const char* realName = name.c_str(); + const auto& ovShape = input_tensor.get_shape(); + std::vector capiShape = getShapeAcceptableByCAPI(ovShape); + OVMS_DataType inputDataType = OVPrecision2CAPI(input_tensor.get_element_type()); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realName, inputDataType, capiShape.data(), capiShape.size())); + const uint32_t NOT_USED_NUM = 0; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, + realName, + reinterpret_cast(input_tensor.data()), + input_tensor.get_byte_size(), + OVMS_BUFFERTYPE_CPU, + NOT_USED_NUM)); + } +#if (OVMS_DUMP_TO_FILE == 1) + dumpOvTensorInput(input, "input"); +#endif + OVMS_InferenceResponse* response = nullptr; + status = OVMS_Inference(cserver, request, &response); + if (nullptr != status) { + uint32_t code = 0; + const char* msg = nullptr; + THROW_IF_CIRCULAR_ERR(OVMS_StatusCode(status, &code)); + THROW_IF_CIRCULAR_ERR(OVMS_StatusDetails(status, &msg)); + std::stringstream ss; + ss << "Inference in OVMSAdapter failed: "; + ss << msg << " code: " << code; + LOG(INFO) << ss.str(); + OVMS_StatusDelete(status); + return output; + } + CREATE_GUARD(responseGuard, OVMS_InferenceResponse, response); + uint32_t outputCount = 42; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutputCount(response, &outputCount)); + uint32_t parameterCount = 42; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseParameterCount(response, ¶meterCount)); + const void* voutputData; + size_t bytesize = 42; + OVMS_DataType datatype = (OVMS_DataType)199; + const int64_t* shape{nullptr}; + size_t dimCount = 42; + OVMS_BufferType bufferType = (OVMS_BufferType)199; + uint32_t deviceId = 42; + const char* outputName{nullptr}; + for (size_t i = 0; i < outputCount; ++i) { + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutput(response, i, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); + output.emplace(outputName, std::move(makeOvTensor(datatype, shape, dimCount, voutputData, bytesize))); + } +#if (OVMS_DUMP_TO_FILE == 1) + dumpOvTensorInput(output, "output"); +#endif + return output; +} + +void OVMSInferenceAdapter::loadModel(const std::shared_ptr& model, ov::Core& core, + const std::string& device, const ov::AnyMap& compilationConfig, size_t max_num_requests) { + OVMS_ServableMetadata* servableMetadata = nullptr; + ASSERT_CAPI_STATUS_NULL(OVMS_GetServableMetadata(cserver, servableName.c_str(), servableVersion, &servableMetadata)); + CREATE_GUARD(metadataGuard, OVMS_ServableMetadata, servableMetadata); + uint32_t inputCount = 0; + uint32_t outputCount = 0; + ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataInputCount(servableMetadata, &inputCount)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataOutputCount(servableMetadata, &outputCount)); + + uint32_t id = 0; + OVMS_DataType datatype = (OVMS_DataType)199; + int64_t* shapeMin{nullptr}; + int64_t* shapeMax{nullptr}; + size_t dimCount = 42; + const char* tensorName{nullptr}; + for (id = 0; id < inputCount; ++id) { + ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataInput(servableMetadata, id, &tensorName, &datatype, &dimCount, &shapeMin, &shapeMax)); + inputNames.emplace_back(tensorName); + shape_min_max_t inputMinMax; + for (size_t i = 0; i < dimCount; ++i) { + inputMinMax.first.emplace_back(shapeMin[i]); + inputMinMax.second.emplace_back(shapeMax[i]); + } + this->inShapesMinMaxes.insert({tensorName, std::move(inputMinMax)}); + this->inputDatatypes.insert({tensorName, CAPI2OVPrecision(datatype)}); + } + for (id = 0; id < outputCount; ++id) { + ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataOutput(servableMetadata, id, &tensorName, &datatype, &dimCount, &shapeMin, &shapeMax)); + outputNames.emplace_back(tensorName); + shape_min_max_t outputMinMax; + for (size_t i = 0; i < dimCount; ++i) { + outputMinMax.first.emplace_back(shapeMin[i]); + outputMinMax.second.emplace_back(shapeMax[i]); + } + this->outShapesMinMaxes.insert({tensorName, std::move(outputMinMax)}); + this->outputDatatypes.insert({tensorName, CAPI2OVPrecision(datatype)}); + } + const ov::AnyMap* servableMetadataRtInfo; + ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataInfo(servableMetadata, reinterpret_cast(&servableMetadataRtInfo))); + try { + if ((*servableMetadataRtInfo).count("model_info") == 0) { + this->modelConfig = ov::AnyMap{}; + } else { + this->modelConfig = (*servableMetadataRtInfo).at("model_info").as(); + } + } catch (const std::exception& e) { + LOG(INFO) << "Exception occurred while accessing model_info: " << e.what(); + this->modelConfig = ov::AnyMap{}; + } +} + +ov::element::Type_t OVMSInferenceAdapter::getInputDatatype(const std::string& inputName) const { + return inputDatatypes.at(inputName); +} + +ov::element::Type_t OVMSInferenceAdapter::getOutputDatatype(const std::string& outputName) const { + return outputDatatypes.at(outputName); +} + +ov::PartialShape OVMSInferenceAdapter::getInputShape(const std::string& inputName) const { + auto it = inShapesMinMaxes.find(inputName); + if (it == inShapesMinMaxes.end()) { + LOG(INFO) << "Could not find input:" << inputName; + throw std::runtime_error(std::string("Adapter could not find input:") + inputName); + } + + ov::PartialShape ovShape; + const auto& [minBorder, maxBorder] = it->second; + ovShape.reserve(minBorder.size()); + for (size_t i = 0; i < minBorder.size(); ++i) { + ovShape.emplace_back(ov::Dimension{minBorder[i], maxBorder[i]}); + } + return ovShape; +} + +ov::PartialShape OVMSInferenceAdapter::getOutputShape(const std::string& outputName) const { + auto it = outShapesMinMaxes.find(outputName); + if (it == outShapesMinMaxes.end()) { + LOG(INFO) << "Could not find output:" << outputName; + throw std::runtime_error(std::string("Adapter could not find output:") + outputName); + } + + ov::PartialShape ovShape; + const auto& [minBorder, maxBorder] = it->second; + ovShape.reserve(minBorder.size()); + for (size_t i = 0; i < minBorder.size(); ++i) { + ovShape.emplace_back(ov::Dimension{minBorder[i], maxBorder[i]}); + } + return ovShape; +} + +std::vector OVMSInferenceAdapter::getInputNames() const { return inputNames; } + +std::vector OVMSInferenceAdapter::getOutputNames() const { return outputNames; } + +const ov::AnyMap& OVMSInferenceAdapter::getModelConfig() const { + return modelConfig; +} + +void OVMSInferenceAdapter::inferAsync(const InferenceInput& input, CallbackData callback_args) { + throw std::runtime_error("Model_api async calls not implemented exception."); +} + +void OVMSInferenceAdapter::setCallback(std::function callback) { + throw std::runtime_error("Model_api async calls not implemented exception."); +} + +bool OVMSInferenceAdapter::isReady() { + throw std::runtime_error("Model_api async calls not implemented exception."); + return false; +} + +void OVMSInferenceAdapter::awaitAll() { + throw std::runtime_error("Model_api async calls not implemented exception."); +} + +void OVMSInferenceAdapter::awaitAny() { + throw std::runtime_error("Model_api async calls not implemented exception."); +} + +size_t OVMSInferenceAdapter::getNumAsyncExecutors() const { + throw std::runtime_error("Model_api async calls not implemented exception."); + return 0; +} + +static OVMS_DataType OVPrecision2CAPI(ov::element::Type_t datatype) { + static std::unordered_map precisionMap{ + {ov::element::Type_t::f64, OVMS_DATATYPE_FP64}, + {ov::element::Type_t::f32, OVMS_DATATYPE_FP32}, + {ov::element::Type_t::f16, OVMS_DATATYPE_FP16}, + {ov::element::Type_t::i64, OVMS_DATATYPE_I64}, + {ov::element::Type_t::i32, OVMS_DATATYPE_I32}, + {ov::element::Type_t::i16, OVMS_DATATYPE_I16}, + {ov::element::Type_t::i8, OVMS_DATATYPE_I8}, + {ov::element::Type_t::i4, OVMS_DATATYPE_I4}, + {ov::element::Type_t::u64, OVMS_DATATYPE_U64}, + {ov::element::Type_t::u32, OVMS_DATATYPE_U32}, + {ov::element::Type_t::u16, OVMS_DATATYPE_U16}, + {ov::element::Type_t::u8, OVMS_DATATYPE_U8}, + {ov::element::Type_t::u4, OVMS_DATATYPE_U4}, + {ov::element::Type_t::u1, OVMS_DATATYPE_U1}, + {ov::element::Type_t::boolean, OVMS_DATATYPE_BOOL}, + {ov::element::Type_t::bf16, OVMS_DATATYPE_BF16}, + {ov::element::Type_t::dynamic, OVMS_DATATYPE_DYNAMIC}, + {ov::element::Type_t::string, OVMS_DATATYPE_STRING}, + }; + auto it = precisionMap.find(datatype); + if (it == precisionMap.end()) { + return OVMS_DATATYPE_UNDEFINED; + } + return it->second; +} + +static ov::element::Type_t CAPI2OVPrecision(OVMS_DataType datatype) { + static std::unordered_map precisionMap{ + {OVMS_DATATYPE_FP64, ov::element::Type_t::f64}, + {OVMS_DATATYPE_FP32, ov::element::Type_t::f32}, + {OVMS_DATATYPE_FP16, ov::element::Type_t::f16}, + {OVMS_DATATYPE_I64, ov::element::Type_t::i64}, + {OVMS_DATATYPE_I32, ov::element::Type_t::i32}, + {OVMS_DATATYPE_I16, ov::element::Type_t::i16}, + {OVMS_DATATYPE_I8, ov::element::Type_t::i8}, + {OVMS_DATATYPE_I4, ov::element::Type_t::i4}, + {OVMS_DATATYPE_U64, ov::element::Type_t::u64}, + {OVMS_DATATYPE_U32, ov::element::Type_t::u32}, + {OVMS_DATATYPE_U16, ov::element::Type_t::u16}, + {OVMS_DATATYPE_U8, ov::element::Type_t::u8}, + {OVMS_DATATYPE_U4, ov::element::Type_t::u4}, + {OVMS_DATATYPE_U1, ov::element::Type_t::u1}, + {OVMS_DATATYPE_BOOL, ov::element::Type_t::boolean}, + {OVMS_DATATYPE_BF16, ov::element::Type_t::bf16}, + {OVMS_DATATYPE_UNDEFINED, ov::element::Type_t::dynamic}, + {OVMS_DATATYPE_DYNAMIC, ov::element::Type_t::dynamic}, + }; + auto it = precisionMap.find(datatype); + if (it == precisionMap.end()) { + return ov::element::Type_t::dynamic; + } + return it->second; +} + +static ov::Tensor makeOvTensor(OVMS_DataType datatype, const int64_t* shape, size_t dimCount, const void* voutputData, size_t bytesize) { + ov::Shape ovShape; + for (size_t i = 0; i < dimCount; ++i) { + ovShape.push_back(shape[i]); + } + ov::Tensor output(CAPI2OVPrecision(datatype), ovShape); + std::memcpy(output.data(), voutputData, bytesize); + return output; +} + +#pragma GCC diagnostic pop +} // namespace mediapipe::ovms diff --git a/src/mediapipe_calculators/ovms/modelapiovmsadapter.hpp b/src/mediapipe_calculators/ovms/modelapiovmsadapter.hpp new file mode 100644 index 0000000000..eb5a436d7a --- /dev/null +++ b/src/mediapipe_calculators/ovms/modelapiovmsadapter.hpp @@ -0,0 +1,84 @@ +#pragma once +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include + +#include // model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h +#include + +#include "ovms.h" // NOLINT + +class OVMS_Server_; +typedef struct OVMS_Server_ OVMS_Server; + +namespace mediapipe::ovms { + +using InferenceOutput = std::map; +using InferenceInput = std::map; + +using shape_border_t = std::vector; +using shape_min_max_t = std::pair; +using shapes_min_max_t = std::unordered_map; +class OVMSInferenceAdapter : public ::InferenceAdapter { + OVMS_Server* cserver{nullptr}; + const std::string servableName; + uint32_t servableVersion; + std::vector inputNames; + std::vector outputNames; + shapes_min_max_t inShapesMinMaxes; + shapes_min_max_t outShapesMinMaxes; + std::unordered_map inputDatatypes; + std::unordered_map outputDatatypes; + ov::AnyMap modelConfig; + +public: + // TODO Windows: Fix definition in header - does not compile in cpp. + OVMSInferenceAdapter(const std::string& servableName, uint32_t servableVersion = 0, OVMS_Server* server = nullptr) : + servableName(servableName), + servableVersion(servableVersion) { + if (nullptr != server) { + this->cserver = server; + } else { + OVMS_ServerNew(&this->cserver); + } + } + virtual ~OVMSInferenceAdapter(); + InferenceOutput infer(const InferenceInput& input) override; + void infer(const InferenceInput& input, InferenceOutput& output) override; + void loadModel(const std::shared_ptr& model, ov::Core& core, + const std::string& device, const ov::AnyMap& compilationConfig, size_t max_num_requests = 1) override; + void inferAsync(const InferenceInput& input, const CallbackData callback_args) override; + void setCallback(std::function callback); + bool isReady(); + void awaitAll(); + void awaitAny(); + size_t getNumAsyncExecutors() const; + ov::PartialShape getInputShape(const std::string& inputName) const override; + ov::PartialShape getOutputShape(const std::string& outputName) const override; + ov::element::Type_t getInputDatatype(const std::string& inputName) const override; + ov::element::Type_t getOutputDatatype(const std::string& outputName) const override; + std::vector getInputNames() const override; + std::vector getOutputNames() const override; + const ov::AnyMap& getModelConfig() const override; +}; +} // namespace mediapipe::ovms diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc new file mode 100644 index 0000000000..f24b1f37c5 --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc @@ -0,0 +1,390 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include + +#include // model_api/model_api/cpp/adapters/include/adapters/inference_adapter.h +#include +#include +#include + +#include "ovms.h" // NOLINT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "openvinoinferencecalculator.h" +#include "openvinoinferencecalculatoroptions.h" +#include "openvinoinferenceutils.h" +#include "src/mediapipe_calculators/ovms/openvinoinferencecalculator.pb.h" +#if (OVMS_DUMP_TO_FILE == 1) +#include "openvinoinferencedumputils.h" +#endif +#pragma GCC diagnostic pop +namespace mediapipe { +using std::endl; + +static Tensor::ElementType OVType2MPType(ov::element::Type_t precision) { + static std::unordered_map precisionMap{ + {ov::element::Type_t::f32, Tensor::ElementType::kFloat32}, + {ov::element::Type_t::f16, Tensor::ElementType::kFloat16}, + {ov::element::Type_t::i32, Tensor::ElementType::kInt32}, + {ov::element::Type_t::i8, Tensor::ElementType::kInt8}, + {ov::element::Type_t::u8, Tensor::ElementType::kUInt8}, + {ov::element::Type_t::boolean, Tensor::ElementType::kBool}, + }; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + return Tensor::ElementType::kNone; + } + return it->second; +} + +static ov::element::Type_t MPType2OVType(Tensor::ElementType precision) { + static std::unordered_map precisionMap{ + {Tensor::ElementType::kFloat32, ov::element::Type_t::f32}, + {Tensor::ElementType::kFloat16, ov::element::Type_t::f16}, + {Tensor::ElementType::kInt32, ov::element::Type_t::i32}, + {Tensor::ElementType::kInt8, ov::element::Type_t::i8}, + {Tensor::ElementType::kUInt8, ov::element::Type_t::u8}, + {Tensor::ElementType::kBool, ov::element::Type_t::boolean}, + }; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + return ov::element::Type_t::dynamic; + } + return it->second; +} + +static ov::Tensor convertMPTensor2OVTensor(const Tensor& inputTensor) { + void* data; + switch (inputTensor.element_type()) { + case Tensor::ElementType::kFloat32: + case Tensor::ElementType::kFloat16: + data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); + break; + case Tensor::ElementType::kUInt8: + data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); + break; + case Tensor::ElementType::kInt8: + data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); + break; + case Tensor::ElementType::kInt32: + data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); + break; + case Tensor::ElementType::kBool: + data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); + break; + default: + data = reinterpret_cast(const_cast(inputTensor.GetCpuReadView().buffer())); + break; + } + auto datatype = MPType2OVType(inputTensor.element_type()); + if (datatype == ov::element::Type_t::dynamic) { + LOG(INFO) << "Not supported precision for Mediapipe tensor deserialization"; + throw std::runtime_error("Not supported precision for Mediapipe tensor deserialization"); + } + ov::Shape shape; + for (const auto& dim : inputTensor.shape().dims) { + shape.emplace_back(dim); + } + ov::Tensor result(datatype, shape, data); + return result; +} + +static Tensor convertOVTensor2MPTensor(const ov::Tensor& inputTensor) { + std::vector rawShape; + for (size_t i = 0; i < inputTensor.get_shape().size(); i++) { + rawShape.emplace_back(inputTensor.get_shape()[i]); + } + Tensor::Shape shape{rawShape}; + auto datatype = OVType2MPType(inputTensor.get_element_type()); + if (datatype == mediapipe::Tensor::ElementType::kNone) { + LOG(INFO) << "Not supported precision for Mediapipe tensor serialization: " << inputTensor.get_element_type(); + throw std::runtime_error("Not supported precision for Mediapipe tensor serialization"); + } + Tensor outputTensor(datatype, shape); + void* data; + switch (inputTensor.get_element_type()) { + case ov::element::Type_t::f32: + case ov::element::Type_t::f16: + data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); + break; + case ov::element::Type_t::u8: + data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); + break; + case ov::element::Type_t::i8: + data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); + break; + case ov::element::Type_t::i32: + data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); + break; + case ov::element::Type_t::boolean: + data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); + break; + default: + data = reinterpret_cast(const_cast(outputTensor.GetCpuWriteView().buffer())); + break; + } + std::memcpy(data, inputTensor.data(), inputTensor.get_byte_size()); + return outputTensor; +} + +absl::Status OpenVINOInferenceCalculator::GetContract(CalculatorContract* cc) { + LOG(INFO) << "OpenVINOInferenceCalculator GetContract start"; + RET_CHECK(!cc->Inputs().GetTags().empty()); + RET_CHECK(!cc->Outputs().GetTags().empty()); + RET_CHECK(cc->InputSidePackets().HasTag(SESSION_TAG)); + RET_CHECK(ValidateCalculatorSettings(cc)); + + for (const std::string& tag : cc->Inputs().GetTags()) { + if (startsWith(tag, OVTENSORS_TAG)) { + LOG(INFO) << "setting input tag:" << tag << " to OVTensors"; + cc->Inputs().Tag(tag).Set>(); + } else if (startsWith(tag, OVTENSOR_TAG)) { + LOG(INFO) << "setting input tag:" << tag << " to OVTensor"; + cc->Inputs().Tag(tag).Set(); + } else if (startsWith(tag, MPTENSORS_TAG)) { + LOG(INFO) << "setting input tag:" << tag << " to MPTensors"; + cc->Inputs().Tag(tag).Set>(); + } else if (startsWith(tag, MPTENSOR_TAG)) { + LOG(INFO) << "setting input tag:" << tag << " to MPTensor"; + cc->Inputs().Tag(tag).Set(); + } else { + LOG(INFO) << "setting input tag:" << tag << " to OVTensor"; + cc->Inputs().Tag(tag).Set(); + } + } + for (const std::string& tag : cc->Outputs().GetTags()) { + if (startsWith(tag, OVTENSORS_TAG)) { + LOG(INFO) << "setting output tag:" << tag << " to std::vector"; + cc->Outputs().Tag(tag).Set>(); + } else if (startsWith(tag, OVTENSOR_TAG)) { + LOG(INFO) << "setting output tag:" << tag << " to OVTensor"; + cc->Outputs().Tag(tag).Set(); + } else if (startsWith(tag, MPTENSORS_TAG)) { + LOG(INFO) << "setting output tag:" << tag << " to std::vector"; + cc->Outputs().Tag(tag).Set>(); + } else if (startsWith(tag, MPTENSOR_TAG)) { + LOG(INFO) << "setting output tag:" << tag << " to MPTensor"; + cc->Outputs().Tag(tag).Set(); + } else { + LOG(INFO) << "setting output tag:" << tag << " to OVTensor"; + cc->Outputs().Tag(tag).Set(); + } + } + cc->InputSidePackets().Tag(SESSION_TAG.c_str()).Set>(); + LOG(INFO) << "OpenVINOInferenceCalculator GetContract end"; + return absl::OkStatus(); +} + +absl::Status OpenVINOInferenceCalculator::Close(CalculatorContext* cc) { + LOG(INFO) << "OpenVINOInferenceCalculator Close"; + return absl::OkStatus(); +} + +absl::Status OpenVINOInferenceCalculator::Open(CalculatorContext* cc) { + LOG(INFO) << "OpenVINOInferenceCalculator Open start"; + session = cc->InputSidePackets() + .Tag(SESSION_TAG.c_str()) + .Get>(); + for (CollectionItemId id = cc->Inputs().BeginId(); + id < cc->Inputs().EndId(); ++id) { + if (!cc->Inputs().Get(id).Header().IsEmpty()) { + cc->Outputs().Get(id).SetHeader(cc->Inputs().Get(id).Header()); + } + } + if (cc->OutputSidePackets().NumEntries() != 0) { + for (CollectionItemId id = cc->InputSidePackets().BeginId(); + id < cc->InputSidePackets().EndId(); ++id) { + cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); + } + } + const auto& options = cc->Options(); + for (const auto& [key, value] : options.tag_to_output_tensor_names()) { + outputNameToTag[value] = key; + } + + auto& input_list = options.input_order_list(); + input_order_list.clear(); + for (int i = 0; i < input_list.size(); i++) { + input_order_list.push_back(input_list[i]); + } + auto& output_list = options.output_order_list(); + output_order_list.clear(); + for (int i = 0; i < output_list.size(); i++) { + output_order_list.push_back(output_list[i]); + } + + cc->SetOffset(TimestampDiff(0)); + LOG(INFO) << "OpenVINOInferenceCalculator Open end"; + return absl::OkStatus(); +} + +absl::Status OpenVINOInferenceCalculator::Process(CalculatorContext* cc) { + LOG(INFO) << "OpenVINOInferenceCalculator process start"; + if (cc->Inputs().NumEntries() == 0) { + return tool::StatusStop(); + } + + const auto& options = cc->Options(); + const auto& inputTagInputMap = options.tag_to_input_tensor_names(); + ::InferenceInput input; + ::InferenceOutput output; + for (const std::string& tag : cc->Inputs().GetTags()) { + if (cc->Inputs().Tag(tag).IsEmpty()) { + LOG(INFO) << "OpenVINOInferenceCalculator expects all input packets at the same time for each call to Process()."; + RET_CHECK(false); + } + const char* realInputName{nullptr}; + auto it = inputTagInputMap.find(tag); + if (it == inputTagInputMap.end()) { + realInputName = tag.c_str(); + } else { + realInputName = it->second.c_str(); + } +#define DESERIALIZE_TENSORS(TYPE, DESERIALIZE_FUN) \ + auto& packet = cc->Inputs().Tag(tag).Get>(); \ + if (packet.size() != this->input_order_list.size()) { \ + LOG(INFO) << "input_order_list size does not match the input vector size."; \ + RET_CHECK(false); \ + } \ + for (size_t i = 0; i < this->input_order_list.size(); i++) { \ + auto& tensor = packet[i]; \ + input[this->input_order_list[i]] = DESERIALIZE_FUN(tensor); \ + } + + try { + if (startsWith(tag, OVTENSORS_TAG)) { + DESERIALIZE_TENSORS(ov::Tensor, ); + } else if (startsWith(tag, MPTENSORS_TAG)) { + DESERIALIZE_TENSORS(Tensor, convertMPTensor2OVTensor); + } else if (startsWith(tag, OVTENSOR_TAG)) { + auto& packet = cc->Inputs().Tag(tag).Get(); + input[realInputName] = packet; + } else if (startsWith(tag, MPTENSOR_TAG)) { + auto& packet = cc->Inputs().Tag(tag).Get(); + input[realInputName] = convertMPTensor2OVTensor(packet); + } else { + auto& packet = cc->Inputs().Tag(tag).Get(); + input[realInputName] = packet; + } + } catch (const std::runtime_error& e) { + LOG(INFO) << "Failed to deserialize tensor error:" << e.what(); + RET_CHECK(false); + } + } +#if (OVMS_DUMP_TO_FILE == 1) + dumpOvTensorInput(input, "input"); +#endif + try { + output = session->infer(input); +#if (OVMS_DUMP_TO_FILE == 1) + dumpOvTensorInput(output, "output"); +#endif + } catch (const std::exception& e) { + LOG(INFO) << "Caught exception from session infer():" << e.what(); + RET_CHECK(false); + } catch (...) { + LOG(INFO) << "Caught unknown exception from session infer()"; + RET_CHECK(false); + } + auto outputsCount = output.size(); + RET_CHECK(outputsCount >= cc->Outputs().GetTags().size()); + LOG(INFO) << "output tags size: " << cc->Outputs().GetTags().size(); + for (const auto& tag : cc->Outputs().GetTags()) { + LOG(INFO) << "Processing tag: " << tag; + std::string tensorName; + auto tensorIt = output.begin(); + + if (!IsVectorTag(tag)) { + auto it = options.tag_to_output_tensor_names().find(tag); + if (it == options.tag_to_output_tensor_names().end()) { + tensorName = tag; + } else { + tensorName = it->second; + } + tensorIt = output.find(tensorName); + if (tensorIt == output.end()) { + LOG(INFO) << "Could not find: " << tensorName << " in inference output"; + RET_CHECK(false); + } + } + + try { +#define SERIALIZE_TENSORS(TYPE, SESERIALIZE_FUN) \ + auto tensors = std::make_unique>(); \ + if (output.size() > 1 && this->output_order_list.size() != this->output_order_list.size()) { \ + LOG(INFO) << "output_order_list not set properly in options for multiple outputs."; \ + RET_CHECK(false); \ + } \ + if (this->output_order_list.size() > 0) { \ + for (const auto& tensorName : this->output_order_list) { \ + tensorIt = output.find(tensorName); \ + if (tensorIt == output.end()) { \ + LOG(INFO) << "Could not find: " << tensorName << " in inference output"; \ + RET_CHECK(false); \ + } \ + tensors->emplace_back(SESERIALIZE_FUN(tensorIt->second)); \ + } \ + } else { \ + for (auto& [name, tensor] : output) { \ + tensors->emplace_back(SESERIALIZE_FUN(tensor)); \ + } \ + } \ + cc->Outputs().Tag(tag).Add( \ + tensors.release(), \ + cc->InputTimestamp()); + + if (startsWith(tag, OVTENSORS_TAG)) { + LOG(INFO) << "OVMS calculator will process vector"; + SERIALIZE_TENSORS(ov::Tensor, ) + } else if (startsWith(tag, MPTENSORS_TAG)) { + LOG(INFO) << "OVMS calculator will process vector"; + SERIALIZE_TENSORS(Tensor, convertOVTensor2MPTensor) + } else if (startsWith(tag, OVTENSOR_TAG)) { + LOG(INFO) << "OVMS calculator will process ov::Tensor"; + cc->Outputs().Tag(tag).Add( + new ov::Tensor(tensorIt->second), + cc->InputTimestamp()); + } else if (startsWith(tag, MPTENSOR_TAG)) { + LOG(INFO) << "OVMS calculator will process mediapipe::Tensor"; + cc->Outputs().Tag(tag).Add( + new Tensor(convertOVTensor2MPTensor(tensorIt->second)), + cc->InputTimestamp()); + } else { + LOG(INFO) << "OVMS calculator will process ov::Tensor"; + cc->Outputs().Tag(tag).Add( + new ov::Tensor(tensorIt->second), + cc->InputTimestamp()); + } + } catch (const std::runtime_error& e) { + LOG(INFO) << "Failed to deserialize tensor error:" << e.what(); + RET_CHECK(false); + } + } + LOG(INFO) << "OpenVINOInferenceCalculator process end"; + return absl::OkStatus(); +} + +REGISTER_CALCULATOR(OpenVINOInferenceCalculator); +} // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.h b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.h new file mode 100644 index 0000000000..4963239e04 --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.h @@ -0,0 +1,42 @@ +#pragma once +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#pragma GCC diagnostic pop + +class InferenceAdapter; +namespace mediapipe { +class OpenVINOInferenceCalculator : public CalculatorBase { + std::shared_ptr<::InferenceAdapter> session{nullptr}; + std::unordered_map outputNameToTag; + std::vector input_order_list; + std::vector output_order_list; + +public: + static absl::Status GetContract(CalculatorContract* cc); + absl::Status Close(CalculatorContext* cc) override final; + absl::Status Open(CalculatorContext* cc) override final; + absl::Status Process(CalculatorContext* cc) override final; +}; +} // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.proto b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.proto new file mode 100644 index 0000000000..e5b36112a6 --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.proto @@ -0,0 +1,33 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +syntax = "proto2"; +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message OpenVINOInferenceCalculatorOptions { + extend mediapipe.CalculatorOptions { + // https://github.com/google/mediapipe/issues/634 have to be unique in app + // no rule to obtain this + optional OpenVINOInferenceCalculatorOptions ext = 113473743; + } + map tag_to_input_tensor_names = 1; + map tag_to_output_tensor_names = 2; + // repeated gives optional option by default + repeated string input_order_list = 3; + repeated string output_order_list = 4; +} diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc new file mode 100644 index 0000000000..90f9ca18d8 --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc @@ -0,0 +1,204 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include + +#include "ovms.h" // NOLINT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/calculator_contract.h" +#include "src/mediapipe_calculators/ovms/openvinoinferencecalculator.pb.h" +#pragma GCC diagnostic pop +#include "openvinoinferencecalculatoroptions.h" +#include "openvinoinferenceutils.h" + +namespace mediapipe { + +const char* OvmsLogLevelEnv = "GLOG_minloglevel"; + +static bool ValidateOrderLists(std::set calculatorTags, const google::protobuf::RepeatedPtrField& order_list) { + std::vector inputTypes; + for (const std::string& tag : calculatorTags) { + std::vector tokens = tokenize(tag, ':'); + if (tokens.size() > 0) { + std::string inputType = tokens[0]; + + for (const auto& supportedVectorTag : supportedVectorTags) { + if (startsWith(inputType, supportedVectorTag)) { + if (order_list.size() < 1) { + LOG(ERROR) << "OpenVINOInferenceCalculator GetContract error. Order list is requiered for vector types: " << inputType; + return false; + } + } + } + } + } + + return true; +} + +static bool ValidateOrderListsForNonVector(std::set calculatorTags, const google::protobuf::RepeatedPtrField& order_list) { + std::vector inputTypes; + bool vectorTypeExists = false; + for (const std::string& tag : calculatorTags) { + std::vector tokens = tokenize(tag, ':'); + if (tokens.size() > 0) { + std::string inputType = tokens[0]; + if (IsVectorTag(inputType)) { + vectorTypeExists = true; + } + } + } + if (!vectorTypeExists && order_list.size() > 0) { + LOG(ERROR) << "OpenVINOInferenceCalculator GetContract error. Using odrer_list for non vector type. " << order_list[0]; + return false; + } + + return true; +} + +bool IsVectorTag(const std::string& tag) { + for (const auto& supportedVectorTag : supportedVectorTags) { + if (startsWith(tag, supportedVectorTag)) { + return true; + } + } + + return false; +} + +static bool ValidateTagToNames(std::set calculatorTags, const google::protobuf::Map& tags_to_names) { + std::vector inputTypes; + for (const std::string& tag : calculatorTags) { + std::vector tokens = tokenize(tag, ':'); + if (tokens.size() > 0) { + inputTypes.push_back(tokens[0]); + } else { + inputTypes.push_back(tag); + } + } + + for (const auto& [key, value] : tags_to_names) { + bool nameMatch = false; + + for (const auto& supportedTag : supportedTags) { + if (startsWith(key, supportedTag)) { + if (endsWith(key, "S") && !endsWith(supportedTag, "S")) + continue; + + for (const auto& graphInput : inputTypes) { + if (startsWith(graphInput, supportedTag)) { + if (endsWith(graphInput, "S") && !endsWith(supportedTag, "S")) + continue; + nameMatch = true; + break; + } + } + } + + if (tokenize(key, ':').size() == 0) { + for (const auto& graphInput : inputTypes) { + if (key == graphInput) { + nameMatch = true; + break; + } + } + } + + if (nameMatch) { + break; + } + } + + for (const auto& graphInput : inputTypes) { + if (key == graphInput) { + nameMatch = true; + break; + } + } + + if (!nameMatch) { + LOG(ERROR) << "OpenVINOInferenceCalculator GetContract error. Stream names mismatch for tag_to__tensor_names name key " << key; + return false; + } + } + + return true; +} + +static bool ValidateOptions(CalculatorContract* cc) { + const auto& options = cc->Options(); + + if (options.tag_to_output_tensor_names().size() > 0 && options.output_order_list().size() > 0) { + LOG(ERROR) << "OpenVINOInferenceCalculator GetContract error. Use tag_to_output_tensor_names or output_order_list not both at once."; + return false; + } + + if (options.tag_to_input_tensor_names().size() > 0 && options.input_order_list().size() > 0) { + LOG(ERROR) << "OpenVINOInferenceCalculator GetContract error. Use tag_to_input_tensor_names or input_order_list not both at once."; + return false; + } + + return true; +} + +bool ValidateCalculatorSettings(CalculatorContract* cc) { + if (!ValidateOptions(cc)) { + LOG(INFO) << "OpenVINOInferenceCalculator ValidateOptions failed."; + return false; + } + + // Skip deep validation when INFO log level to avoid performance impact. + if (StringToLogLevel(std::string(std::getenv(OvmsLogLevelEnv) == nullptr ? "" : std::getenv(OvmsLogLevelEnv))) == OVMS_LogLevel::OVMS_LOG_INFO) + return true; + + const auto& options = cc->Options(); + + if (!ValidateOrderListsForNonVector(cc->Inputs().GetTags(), options.input_order_list())) { + LOG(INFO) << "OpenVINOInferenceCalculator ValidateOrderListsForNonVector for inputs failed."; + return false; + } + if (!ValidateOrderListsForNonVector(cc->Outputs().GetTags(), options.output_order_list())) { + LOG(INFO) << "OpenVINOInferenceCalculator ValidateOrderListsForNonVector for outputs failed."; + return false; + } + + if (!ValidateOrderLists(cc->Inputs().GetTags(), options.input_order_list())) { + LOG(INFO) << "OpenVINOInferenceCalculator ValidateOrderLists for inputs failed."; + return false; + } + if (!ValidateOrderLists(cc->Outputs().GetTags(), options.output_order_list())) { + LOG(INFO) << "OpenVINOInferenceCalculator ValidateOrderLists for outputs failed."; + return false; + } + + if (!ValidateTagToNames(cc->Inputs().GetTags(), options.tag_to_input_tensor_names())) { + LOG(INFO) << "OpenVINOInferenceCalculator ValidateInputTagToNames failed."; + return false; + } + if (!ValidateTagToNames(cc->Outputs().GetTags(), options.tag_to_output_tensor_names())) { + LOG(INFO) << "OpenVINOInferenceCalculator ValidateOutputTagToNames failed."; + return false; + } + + LOG(INFO) << "OpenVINOInferenceCalculator ValidateCalculatorSettings passed."; + return true; +} + +} // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h new file mode 100644 index 0000000000..2d4bda3d76 --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h @@ -0,0 +1,42 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/calculator_contract.h" +#include "src/mediapipe_calculators/ovms/openvinoinferencecalculator.pb.h" +#pragma GCC diagnostic pop + +const std::string SESSION_TAG{"SESSION"}; +const std::string OVTENSOR_TAG{"OVTENSOR"}; +const std::string OVTENSORS_TAG{"OVTENSORS"}; +const std::string MPTENSOR_TAG{"TENSOR"}; +const std::string MPTENSORS_TAG{"TENSORS"}; + +const std::vector supportedTags = {SESSION_TAG, OVTENSOR_TAG, OVTENSORS_TAG, MPTENSOR_TAG, MPTENSORS_TAG}; + +const std::vector supportedVectorTags = {OVTENSORS_TAG, MPTENSORS_TAG}; + +namespace mediapipe { + +bool IsVectorTag(const std::string& tag); + +bool ValidateCalculatorSettings(CalculatorContract* cc); + +} // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc new file mode 100644 index 0000000000..75b4a47165 --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc @@ -0,0 +1,148 @@ +//***************************************************************************** +// Copyright 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "openvinoinferencedumputils.h" + +namespace mediapipe { + +std::unordered_map dumpCounters; + +using InferenceInput = std::map; + +#define TYPE_CASE(ENUM, TYPE) \ + case ENUM: { \ + const TYPE* value = reinterpret_cast(tensor.data()); \ + dumpStream << " Tensor: [ "; \ + for (int x = 0; x < tensor.get_size(); x++) { \ + dumpStream << value[x] << " "; \ + } \ + dumpStream << " ]"; \ + break; \ + } + +static std::stringstream dumpOvTensor(const ov::Tensor& tensor) { + std::stringstream dumpStream; + switch (tensor.get_element_type()) { + TYPE_CASE(ov::element::Type_t::f64, double) + TYPE_CASE(ov::element::Type_t::f32, float) + TYPE_CASE(ov::element::Type_t::i64, int64_t) + TYPE_CASE(ov::element::Type_t::i32, int32_t) + TYPE_CASE(ov::element::Type_t::i16, int16_t) + TYPE_CASE(ov::element::Type_t::i8, int8_t) + TYPE_CASE(ov::element::Type_t::u32, uint32_t) + TYPE_CASE(ov::element::Type_t::u16, uint16_t) + TYPE_CASE(ov::element::Type_t::u8, uint8_t) + TYPE_CASE(ov::element::Type_t::boolean, bool) + case ov::element::Type_t::bf16: + case ov::element::Type_t::dynamic: + case ov::element::Type_t::f16: + case ov::element::Type_t::i4: + case ov::element::Type_t::u4: + case ov::element::Type_t::u1: { + dumpStream << " unsupported dump type: [ " << tensor.get_element_type() << " ]"; + break; + } + } + + return dumpStream; +} + +static bool isAbsolutePath(const std::string& path) { + return !path.empty() && (path[0] == '/'); +} + +static std::string joinPath(std::initializer_list segments) { + std::string joined; + + for (const auto& seg : segments) { + if (joined.empty()) { + joined = seg; + } else if (isAbsolutePath(seg)) { + if (joined[joined.size() - 1] == '/') { + joined.append(seg.substr(1)); + } else { + joined.append(seg); + } + } else { + if (joined[joined.size() - 1] != '/') { + joined.append("/"); + } + joined.append(seg); + } + } + + return joined; +} + +static void writeToFile(std::stringstream& stream, std::string name) { + std::ofstream ofs; + ofs.open(name); + ofs << stream.rdbuf(); + ofs.close(); +} + +static std::string getTimestampString() { + auto rawtime = std::make_unique(); + time(rawtime.get()); + struct tm* timeinfo = localtime(rawtime.get()); + auto start = std::chrono::system_clock::now(); + std::stringstream timestampStream; + timestampStream << timeinfo->tm_year << "_" << timeinfo->tm_mon << "_" << timeinfo->tm_mday << "_"; + timestampStream << timeinfo->tm_hour << "_" << timeinfo->tm_min << "_" << timeinfo->tm_sec << "_"; + using namespace std::chrono; + timestampStream << duration_cast(start.time_since_epoch()).count(); + return timestampStream.str(); +} + +static int getAndIncerementCounter(const std::string& name) { + if (dumpCounters.find(name) == dumpCounters.end()) + dumpCounters[name] = 0; + + return dumpCounters[name]++; +} + +static const std::string TIMESTAMP_STRING = getTimestampString(); + +void dumpOvTensorInput(const InferenceInput& input, const std::string& dumpDirectoryName) { + std::stringstream dumpStream; + std::string fname = std::string("./dump"); + fname = joinPath({fname, TIMESTAMP_STRING}); + std::filesystem::create_directories(fname); + fname = joinPath({fname, dumpDirectoryName + std::to_string(getAndIncerementCounter(dumpDirectoryName))}); + for (const auto& [name, inputTensor] : input) { + dumpStream << " Name: " << name; + dumpStream << " Shape: " << inputTensor.get_shape(); + dumpStream << " Type: " << inputTensor.get_element_type(); + dumpStream << " Byte size: " << inputTensor.get_byte_size(); + dumpStream << " Size: " << inputTensor.get_size(); + dumpStream << dumpOvTensor(inputTensor).str(); + } + + std::cout << "Dump filename: " << fname << std::endl; + writeToFile(dumpStream, fname); +} + +} // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinoinferencedumputils.h b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.h new file mode 100644 index 0000000000..fe34d9654b --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.h @@ -0,0 +1,28 @@ +//***************************************************************************** +// Copyright 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +namespace ov { +class Tensor; +} + +namespace mediapipe { + +using InferenceInput = std::map; +void dumpOvTensorInput(const InferenceInput& input, const std::string& dumpDirectoryName); + +} // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinoinferenceutils.cc b/src/mediapipe_calculators/ovms/openvinoinferenceutils.cc new file mode 100644 index 0000000000..41e580a03e --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinoinferenceutils.cc @@ -0,0 +1,91 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include + +#include "ovms.h" // NOLINT +namespace mediapipe { + +// Function from ovms/src/string_utils.h +bool startsWith(const std::string& str, const std::string& prefix) { + auto it = prefix.begin(); + bool sizeCheck = (str.size() >= prefix.size()); + if (!sizeCheck) { + return false; + } + bool allOf = std::all_of(str.begin(), + std::next(str.begin(), prefix.size()), + [&it](const char& c) { + return c == *(it++); + }); + return allOf; +} + +// Function from ovms/src/string_utils.h +std::vector tokenize(const std::string& str, const char delimiter) { + std::vector tokens; + std::string token; + std::istringstream iss(str); + while (std::getline(iss, token, delimiter)) { + tokens.push_back(token); + } + + return tokens; +} + +// Function from ovms/src/string_utils.h +bool endsWith(const std::string& str, const std::string& match) { + auto it = match.begin(); + return str.size() >= match.size() && + std::all_of(std::next(str.begin(), str.size() - match.size()), str.end(), [&it](const char& c) { + return ::tolower(c) == ::tolower(*(it++)); + }); +} + +OVMS_LogLevel StringToLogLevel(const std::string& logLevel) { + if (logLevel == "3") + return OVMS_LOG_ERROR; + if (logLevel == "1") + return OVMS_LOG_DEBUG; + if (logLevel == "0") + return OVMS_LOG_TRACE; + if (logLevel == "2") + return OVMS_LOG_INFO; + + return OVMS_LOG_INFO; +} + +std::string LogLevelToString(OVMS_LogLevel log_level) { + switch (log_level) { + case OVMS_LOG_INFO: + return "INFO"; + case OVMS_LOG_ERROR: + return "ERROR"; + case OVMS_LOG_DEBUG: + return "DEBUG"; + case OVMS_LOG_TRACE: + return "TRACE"; + case OVMS_LOG_WARNING: + return "WARNING"; + } + + return "unsupported"; +} + +} // namespace mediapipe diff --git a/src/tensorflow_type_utils.hpp b/src/mediapipe_calculators/ovms/openvinoinferenceutils.h similarity index 51% rename from src/tensorflow_type_utils.hpp rename to src/mediapipe_calculators/ovms/openvinoinferenceutils.h index fbcbb9ec46..fc1cd385f7 100644 --- a/src/tensorflow_type_utils.hpp +++ b/src/mediapipe_calculators/ovms/openvinoinferenceutils.h @@ -1,5 +1,5 @@ //***************************************************************************** -// Copyright 2026 Intel Corporation +// Copyright 2023 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,24 +13,24 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#pragma once +#include +#include -#pragma warning(push) -#pragma warning(disable : 4624 6001 6385 6386 6326 6011 4457 6308 6387 6246) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#pragma GCC diagnostic pop -#pragma warning(pop) +#include "ovms.h" // NOLINT -#include "precision.hpp" +namespace mediapipe { -using TFSDataType = tensorflow::DataType; // TODO @atobiszei since we dont have TFS now we can rename to TFDataType? +// Function from ovms/src/string_utils.h +bool startsWith(const std::string& str, const std::string& prefix); -namespace ovms { +// Function from ovms/src/string_utils.h +std::vector tokenize(const std::string& str, const char delimiter); -Precision TFSPrecisionToOvmsPrecision(const TFSDataType& datatype); -TFSDataType getPrecisionAsDataType(Precision precision); +// Function from ovms/src/string_utils.h +bool endsWith(const std::string& str, const std::string& match); -} // namespace ovms +OVMS_LogLevel StringToLogLevel(const std::string& logLevel); + +std::string LogLevelToString(OVMS_LogLevel log_level); + +} // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.cc b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.cc new file mode 100644 index 0000000000..c89f0cf746 --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.cc @@ -0,0 +1,209 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "openvinomodelserversessioncalculator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ovms.h" // NOLINT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "modelapiovmsadapter.hpp" +#include "openvinoinferenceutils.h" +#include "src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.pb.h" +#pragma GCC diagnostic pop +namespace mediapipe { +using ovms::OVMSInferenceAdapter; + +const std::string SESSION_TAG{"SESSION"}; +ov::Core UNUSED_OV_CORE; + +#define ASSERT_CIRCULAR_ERR(C_API_CALL) \ + { \ + auto* fatalErr = C_API_CALL; \ + RET_CHECK(fatalErr == nullptr); \ + } + +#define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ + { \ + auto* err = C_API_CALL; \ + if (err != nullptr) { \ + uint32_t code = 0; \ + const char* msg = nullptr; \ + ASSERT_CIRCULAR_ERR(OVMS_StatusCode(err, &code)); \ + ASSERT_CIRCULAR_ERR(OVMS_StatusDetails(err, &msg)); \ + LOG(INFO) << "Error encountred in OVMSCalculator:" << msg << " code: " << code; \ + OVMS_StatusDelete(err); \ + RET_CHECK(nullptr == err); \ + } \ + } + +// Function from ovms/src/string_utils.h +void erase_spaces(std::string& str) { + str.erase(std::remove_if(str.begin(), str.end(), + [](char c) -> bool { + return std::isspace(c, std::locale::classic()); + }), + str.end()); +} + +// Function from ovms/src/string_utils.h +std::optional stou32(const std::string& input) { + std::string str = input; + erase_spaces(str); + + if (str.size() > 0 && str[0] == '-') { + return std::nullopt; + } + + try { + uint64_t val = std::stoul(str); + if (val > std::numeric_limits::max()) { + return std::nullopt; + } + return {static_cast(val)}; + } catch (...) { + return std::nullopt; + } +} + +class SettingsGuard { +public: + OVMS_ServerSettings* serverSettings{nullptr}; + OVMS_ModelsSettings* modelsSettings{nullptr}; + SettingsGuard() { + OVMS_ServerSettingsNew(&serverSettings); + OVMS_ModelsSettingsNew(&modelsSettings); + } + ~SettingsGuard() { + OVMS_ServerSettingsDelete(serverSettings); + OVMS_ModelsSettingsDelete(modelsSettings); + } +}; + +absl::Status OpenVINOModelServerSessionCalculator::GetContract(CalculatorContract* cc) { + LOG(INFO) << "OpenVINOModelServerSessionCalculator GetContract start"; + RET_CHECK(cc->Inputs().GetTags().empty()); + RET_CHECK(cc->Outputs().GetTags().empty()); + cc->OutputSidePackets().Tag(SESSION_TAG.c_str()).Set>(); + const auto& options = cc->Options(); + RET_CHECK(!options.servable_name().empty()); + + OvmsLogLevel = StringToLogLevel(std::string(std::getenv(OvmsLogLevelEnv) == nullptr ? "" : std::getenv(OvmsLogLevelEnv))); + LOG(INFO) << "OpenVINOModelServerSessionCalculator ovms log level setting: " << LogLevelToString(OvmsLogLevel); + LOG(INFO) << "OpenVINOModelServerSessionCalculator GetContract end"; + return absl::OkStatus(); +} + +absl::Status OpenVINOModelServerSessionCalculator::Close(CalculatorContext* cc) { + LOG(INFO) << "OpenVINOModelServerSessionCalculator Close"; + return absl::OkStatus(); +} + +absl::Status OpenVINOModelServerSessionCalculator::Open(CalculatorContext* cc) { + LOG(INFO) << "OpenVINOModelServerSessionCalculator Open start"; + for (CollectionItemId id = cc->Inputs().BeginId(); + id < cc->Inputs().EndId(); ++id) { + if (!cc->Inputs().Get(id).Header().IsEmpty()) { + cc->Outputs().Get(id).SetHeader(cc->Inputs().Get(id).Header()); + } + } + if (cc->OutputSidePackets().NumEntries() != 0) { + for (CollectionItemId id = cc->InputSidePackets().BeginId(); + id < cc->InputSidePackets().EndId(); ++id) { + cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); + } + } + cc->SetOffset(TimestampDiff(0)); + + const auto& options = cc->Options(); + LOG(INFO) << "Will check if we want to start server"; + if (!options.server_config().empty()) { + // Serialize server startup across concurrent calculator instances. + std::unique_lock lk(OpenVINOModelServerSessionCalculator::loadingMtx); + bool isServerReady = false; + bool isServerLive = false; + OVMS_ServerNew(&cserver); + + ASSERT_CAPI_STATUS_NULL(OVMS_ServerLive(cserver, &isServerLive)); + + if (triedToStartOVMS) { + RET_CHECK(isServerLive); + } else if (!isServerLive) { + LOG(INFO) << "Will start new server"; + triedToStartOVMS = true; + SettingsGuard guard; + OVMS_ServerSettingsNew(&guard.serverSettings); + OVMS_ModelsSettingsNew(&guard.modelsSettings); + OVMS_ServerSettingsSetGrpcPort(guard.serverSettings, 9178); + OVMS_ModelsSettingsSetConfigPath(guard.modelsSettings, options.server_config().c_str()); + LOG(INFO) << "state config file:" << options.server_config(); + OVMS_ServerSettingsSetLogLevel(guard.serverSettings, OvmsLogLevel); + + ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, guard.serverSettings, guard.modelsSettings)); + + ASSERT_CAPI_STATUS_NULL(OVMS_ServerReady(cserver, &isServerReady)); + RET_CHECK(isServerReady); + LOG(INFO) << "Server started"; + } + } + const std::string& servableName = options.servable_name(); + const std::string& servableVersionStr = options.servable_version(); + auto servableVersionOpt = stou32(servableVersionStr); + uint32_t servableVersion = servableVersionOpt.value_or(0); + auto session = std::make_shared(servableName, servableVersion); + try { + session->loadModel(nullptr, UNUSED_OV_CORE, "UNUSED", {}); + } catch (const std::exception& e) { + LOG(INFO) << "Caught exception with message: " << e.what(); + return mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC) + << "OpenVINOModelServerSessionCalculator failed to load the model"; + } catch (...) { + LOG(INFO) << "Caught unknown exception"; + RET_CHECK(false); + } + + LOG(INFO) << "OpenVINOModelServerSessionCalculator create adapter"; + cc->OutputSidePackets().Tag(SESSION_TAG.c_str()).Set(MakePacket>(session)); + LOG(INFO) << "OpenVINOModelServerSessionCalculator Open end"; + return absl::OkStatus(); +} + +absl::Status OpenVINOModelServerSessionCalculator::Process(CalculatorContext* cc) { + LOG(INFO) << "OpenVINOModelServerSessionCalculator Process"; + return absl::OkStatus(); +} + +bool OpenVINOModelServerSessionCalculator::triedToStartOVMS = false; +std::mutex OpenVINOModelServerSessionCalculator::loadingMtx; +const char* OpenVINOModelServerSessionCalculator::OvmsLogLevelEnv = "GLOG_minloglevel"; +OVMS_LogLevel OpenVINOModelServerSessionCalculator::OvmsLogLevel = OVMS_LOG_INFO; + +REGISTER_CALCULATOR(OpenVINOModelServerSessionCalculator); +} // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.h b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.h new file mode 100644 index 0000000000..fbbb9f5e73 --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.h @@ -0,0 +1,52 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ovms.h" // NOLINT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "modelapiovmsadapter.hpp" +#include "src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.pb.h" +#pragma GCC diagnostic pop +namespace mediapipe { + +class OpenVINOModelServerSessionCalculator : public CalculatorBase { + std::shared_ptr<::InferenceAdapter> adapter; + std::unordered_map outputNameToTag; + OVMS_Server* cserver{nullptr}; + static bool triedToStartOVMS; + static std::mutex loadingMtx; + +public: + static absl::Status GetContract(CalculatorContract* cc); + absl::Status Close(CalculatorContext* cc) override final; + absl::Status Open(CalculatorContext* cc) override final; + absl::Status Process(CalculatorContext* cc) override final; + static OVMS_LogLevel OvmsLogLevel; + static const char* OvmsLogLevelEnv; +}; + +} // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.proto b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.proto new file mode 100644 index 0000000000..73e7e23fd6 --- /dev/null +++ b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.proto @@ -0,0 +1,35 @@ +//***************************************************************************** +// Copyright 2023 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +syntax = "proto2"; +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message OpenVINOModelServerSessionCalculatorOptions { + extend mediapipe.CalculatorOptions { + // https://github.com/google/mediapipe/issues/634 have to be unique in app + // no rule to obtain this + optional OpenVINOModelServerSessionCalculatorOptions ext = 113473744; + } + required string servable_name = 1; + optional string servable_version = 2; + // service_url: "13.21.212.171:9718" + optional string service_url = 3; + // config_path: "/models/config.json" + // when this field is used ensure that each calculator is using the same file + optional string server_config = 4; +} diff --git a/src/mediapipe_internal/BUILD b/src/mediapipe_internal/BUILD index 83795ca65d..a24975115b 100644 --- a/src/mediapipe_internal/BUILD +++ b/src/mediapipe_internal/BUILD @@ -240,12 +240,8 @@ ovms_cc_library( name = "libovms_mediapipe_runtime_owner", deps = [ ":libovms_mediapipe", - "//src:kfs_graph_executor_impl_runtime", + "//src:kfs_graph_executor_impl", "//src:libovms_mediapipe_http_executor", - "@mediapipe//mediapipe/graphs/holistic_tracking:holistic_tracking_to_render_data", - "@mediapipe//mediapipe/modules/holistic_landmark:holistic_landmark_cpu", - "@mediapipe//mediapipe/calculators/geti/utils:utils", - "@mediapipe//mediapipe/calculators/geti/serialization:calculators", "//src/embeddings:embeddingscalculator_ov", "//src/rerank:rerankcalculator_ov", "//src/llm:llmcalculator", diff --git a/src/mediapipe_internal/mediapipegraphdefinition.cpp b/src/mediapipe_internal/mediapipegraphdefinition.cpp index 332d1ca5de..377468334f 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.cpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.cpp @@ -272,11 +272,6 @@ Status MediapipeGraphDefinition::dryInitializeTest() { Status MediapipeGraphDefinition::validate(const ServableNameChecker& checker) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Started validation of mediapipe: {}", getName()); SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Validation context for mediapipe: {} graph_path: {} subconfig_path: {}", getName(), this->mgconfig.getGraphPath(), this->mgconfig.getSubconfigPath()); -#if defined(OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME) && OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Build flag OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME is enabled for mediapipe: {}", getName()); -#else - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Build flag OVMS_MEDIAPIPE_DISABLE_TF_TENSOR_RUNTIME is disabled for mediapipe: {}", getName()); -#endif if (!this->sidePacketMaps->empty()) { SPDLOG_ERROR("Internal Error: MediaPipe definition is in unexpected state."); return StatusCode::INTERNAL_ERROR; diff --git a/src/tensorflow_type_utils.cpp b/src/tensorflow_type_utils.cpp deleted file mode 100644 index eca0683104..0000000000 --- a/src/tensorflow_type_utils.cpp +++ /dev/null @@ -1,67 +0,0 @@ -//***************************************************************************** -// Copyright 2026 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "tensorflow_type_utils.hpp" - -#include - -#include "precision.hpp" - -namespace ovms { - -Precision TFSPrecisionToOvmsPrecision(const TFSDataType& datatype) { - static std::unordered_map precisionMap{ - {TFSDataType::DT_FLOAT, Precision::FP32}, - {TFSDataType::DT_DOUBLE, Precision::FP64}, - {TFSDataType::DT_HALF, Precision::FP16}, - {TFSDataType::DT_INT64, Precision::I64}, - {TFSDataType::DT_INT32, Precision::I32}, - {TFSDataType::DT_INT16, Precision::I16}, - {TFSDataType::DT_INT8, Precision::I8}, - {TFSDataType::DT_UINT64, Precision::U64}, - {TFSDataType::DT_UINT16, Precision::U16}, - {TFSDataType::DT_UINT8, Precision::U8}, - {TFSDataType::DT_STRING, Precision::STRING}, - {TFSDataType::DT_BOOL, Precision::BOOL}}; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return Precision::UNDEFINED; - } - return it->second; -} - -TFSDataType getPrecisionAsDataType(Precision precision) { - static std::unordered_map precisionMap{ - {Precision::FP64, TFSDataType::DT_DOUBLE}, - {Precision::FP32, TFSDataType::DT_FLOAT}, - {Precision::FP16, TFSDataType::DT_HALF}, - {Precision::I16, TFSDataType::DT_INT16}, - {Precision::U8, TFSDataType::DT_UINT8}, - {Precision::I8, TFSDataType::DT_INT8}, - {Precision::U16, TFSDataType::DT_UINT16}, - {Precision::I32, TFSDataType::DT_INT32}, - {Precision::I64, TFSDataType::DT_INT64}, - {Precision::U32, TFSDataType::DT_UINT32}, - {Precision::U64, TFSDataType::DT_UINT64}, - {Precision::BOOL, TFSDataType::DT_BOOL}, - {Precision::STRING, TFSDataType::DT_STRING}}; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return TFSDataType::DT_INVALID; - } - return it->second; -} - -} // namespace ovms diff --git a/src/test/ensemble_flow_custom_node_tests.cpp b/src/test/ensemble_flow_custom_node_tests.cpp deleted file mode 100644 index 73ab617ab4..0000000000 --- a/src/test/ensemble_flow_custom_node_tests.cpp +++ /dev/null @@ -1,5976 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include -#include -#include -#include -#include -#include - -#pragma warning(push) -#pragma warning(disable : 4624) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#pragma warning(pop) - -#include -#include - -#include - -#include "../cleaner_utils.hpp" -#include "../dags/custom_node.hpp" -#include "../dags/custom_node_library_manager.hpp" -#include "../dags/dl_node.hpp" -#include "../dags/entry_node.hpp" -#include "../dags/exit_node.hpp" -#include "../dags/node_library.hpp" -#include "../dags/node_library_utils.hpp" -#include "../dags/nodestreamidguard.hpp" -#include "../dags/pipeline.hpp" -#include "../dags/pipeline_factory.hpp" -#include "../dags/pipelinedefinition.hpp" -#include "src/execution_context.hpp" -#include "src/metrics/metric_config.hpp" -#include "src/metrics/metric_registry.hpp" -#include "../model.hpp" -#include "../model_metric_reporter.hpp" -#include "../modelinstance.hpp" -#include "../modelinstanceunloadguard.hpp" -#include "../precision.hpp" -#include "../stringutils.hpp" -#include "constructor_enabled_model_manager.hpp" -#include "test_models_configs.hpp" -#include "test_utils.hpp" -#include "light_test_utils.hpp" -#include "test_with_temp_dir.hpp" - -using namespace ovms; - -using tensorflow::serving::PredictRequest; -using tensorflow::serving::PredictResponse; - -class EnsembleFlowCustomNodePipelineExecutionTest : public TestWithTempDir { -protected: - void SetUp() override { - TestWithTempDir::SetUp(); - - reporter = std::make_unique(&this->metricConfig, &this->registry, "example_pipeline_name", 1); - - CustomNodeLibraryManager manager; - ASSERT_EQ(manager.loadLibrary( - this->libraryName, - getGenericFullPathForBazelOut(this->libraryPath)), - StatusCode::OK); - ASSERT_EQ(manager.getLibrary( - this->libraryName, - this->library), - StatusCode::OK); - dagDummyModelOutputTensorInfo = std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(pipelineInputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - } - - template - void prepareRequest(const std::vector& data) { - this->prepareRequest(this->request, data); - } - - template - void prepareRequest(PredictRequest& request, const std::vector& data, const std::string& inputName = pipelineInputName, const shape_t shape = {}) { - tensorflow::TensorProto& proto = (*request.mutable_inputs())[inputName]; - proto.set_dtype(tensorflow::DataTypeToEnum::value); - proto.mutable_tensor_content()->assign((char*)data.data(), data.size() * sizeof(T)); - if (shape.size()) { - for (auto& dim : shape) { - proto.mutable_tensor_shape()->add_dim()->set_size(dim); - } - } else { - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(data.size()); - } - } - - template - std::unique_ptr prepareSingleNodePipelineWithLibraryMock() { - const std::vector inputValues{3.5, 2.1, -0.2}; - auto inputTensorInfo = std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{1, 3}, - Layout{"NC"}); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - this->prepareRequest(inputValues); - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - auto custom_node = std::make_unique( - customNodeName, - createLibraryMock(), - parameters_t{}); - - auto pipeline = std::make_unique(*input_node, *output_node, *this->reporter); - pipeline->connect(*input_node, *custom_node, {{pipelineInputName, customNodeInputName}}); - pipeline->connect(*custom_node, *output_node, {{customNodeOutputName, pipelineOutputName}}); - - pipeline->push(std::move(input_node)); - pipeline->push(std::move(custom_node)); - pipeline->push(std::move(output_node)); - return pipeline; - } - - template - void checkResponse(std::vector data, std::function op) { - this->checkResponse(this->pipelineOutputName, data, op); - } - - template - void checkResponse(const std::string& outputName, std::vector data, std::function op) { - this->checkResponse(outputName, this->response, data, op); - } - - template - void checkResponse(const std::string& outputName, const PredictResponse& response, const std::vector& data, const shape_t& shape) { - ASSERT_TRUE(response.outputs().contains(outputName)) << outputName; - const auto& proto = response.outputs().at(outputName); - - ASSERT_EQ(proto.tensor_content().size(), data.size() * sizeof(T)); - ASSERT_EQ(proto.tensor_shape().dim_size(), shape.size()); - for (size_t i = 0; i < shape.size(); ++i) { - ASSERT_EQ(proto.tensor_shape().dim(i).size(), shape[i]); - } - - auto* ptr = reinterpret_cast(proto.tensor_content().c_str()); - const std::vector actual(ptr, ptr + data.size()); - for (size_t i = 0; i < actual.size(); i++) { - EXPECT_NEAR(actual[i], data[i], 0.001) << " i is: " << i; - } - } - - template - void checkResponse(const std::string& outputName, const PredictResponse& response, std::vector data, std::function op) { - std::transform(data.begin(), data.end(), data.begin(), op); - ASSERT_TRUE(response.outputs().contains(outputName)); - const auto& proto = response.outputs().at(outputName); - - ASSERT_EQ(proto.tensor_content().size(), data.size() * sizeof(T)); - ASSERT_EQ(proto.tensor_shape().dim_size(), 2); - ASSERT_EQ(proto.tensor_shape().dim(0).size(), 1); - ASSERT_EQ(proto.tensor_shape().dim(1).size(), data.size()); - - auto* ptr = reinterpret_cast(proto.tensor_content().c_str()); - - const std::vector actual(ptr, ptr + data.size()); - - for (size_t i = 0; i < actual.size(); i++) { - EXPECT_NEAR(actual[i], data[i], 0.001); - } - } - - PredictRequest request; - PredictResponse response; - MetricRegistry registry; - MetricConfig metricConfig; - std::unique_ptr reporter; - - NodeLibrary library; - - const std::string customNodeName = "add_sub_node"; - const std::string libraryName = "add_sub_lib"; - const std::string libraryPath = "/ovms/bazel-bin/src/lib_node_add_sub.so"; - const std::string customNodeInputName = "input_numbers"; - const std::string customNodeOutputName = "output_numbers"; - static constexpr const char* pipelineInputName = "pipeline_input"; - const std::string pipelineOutputName = "pipeline_output"; - std::shared_ptr dagDummyModelOutputTensorInfo; - std::shared_ptr dagDummyModelInputTensorInfo; -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, AddSubCustomNode) { - // Most basic configuration, just process single add-sub custom node pipeline request - // input add-sub output - // O------->O------->O - const std::vector inputValues{3.2, 5.7, -2.4}; - this->prepareRequest(inputValues); - - const float addValue = 2.5; - const float subValue = 4.8; - - auto inputTensorInfo = std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{1, 3}, - Layout{"NC"}); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto tensorInfo = std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{1, 3}, - Layout{"NC"}); - const tensor_map_t outputsInfo{{pipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - auto custom_node = std::make_unique(customNodeName, library, - parameters_t{ - {"add_value", std::to_string(addValue)}, - {"sub_value", std::to_string(subValue)}}); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *custom_node, {{pipelineInputName, customNodeInputName}}); - pipeline.connect(*custom_node, *output_node, {{customNodeOutputName, pipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(custom_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(response.outputs().size(), 1); - - this->checkResponse(inputValues, [addValue, subValue](float value) -> float { - return value + addValue - subValue; - }); -} - -class EnsembleFlowCustomNodeAndDemultiplexerGatherPipelineExecutionTest : public EnsembleFlowCustomNodePipelineExecutionTest { -protected: - ConstructorEnabledModelManager modelManager; - ModelConfig config = DUMMY_MODEL_CONFIG; - CustomNodeLibraryManager manager; - NodeLibrary differentOpsLibrary; - NodeLibrary chooseMaxLibrary; - const std::string differentOpsLibraryName{"different_ops"}; - const std::string chooseMaxLibraryName{"choose_max"}; - const std::string differentOpsLibraryPath{"/ovms/bazel-bin/src/lib_node_perform_different_operations.so"}; - const std::string chooseMaxLibraryPath{"/ovms/bazel-bin/src/lib_node_choose_maximum.so"}; - const std::string pipelineInputName = "pipeline_input"; - const std::string pipelineOutputName = "pipeline_output"; - const std::string pipelineFactorsName = "pipeline_factors"; - const std::string chooseMaxInputName = "input_tensors"; - const std::string chooseMaxOutputName = "maximum_tensor"; - const std::string differentOpsInputName = "input_numbers"; - const std::string differentOpsFactorsInputName = "op_factors"; - const std::string differentOpsOutputName = "different_ops_results"; - const std::string differentOpsFactorsOutputName = "different_ops_factors"; - const std::unordered_map differentOpsOutputAlias{{differentOpsOutputName, differentOpsOutputName}}; - const std::unordered_map chooseMaxOutputAlias{{chooseMaxOutputName, chooseMaxOutputName}}; - const std::string dummyNodeName = "dummy"; - const std::string differentOpsNodeName{"different-ops-node"}; - const std::string chooseMaxNodeName{"choose-max-node"}; - const int32_t demultiplyCount = 4; // different ops library has (1,4,10) as output - - void SetUp() override { - EnsembleFlowCustomNodePipelineExecutionTest::SetUp(); - // increasing default nireq == 1 to speed up the tests - // in multilayered demultiplication we still will have more than - // 16 concurrent inferences - config.setNireq(16); - ASSERT_EQ(modelManager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - ASSERT_EQ(manager.loadLibrary( - differentOpsLibraryName, - getGenericFullPathForBazelOut(differentOpsLibraryPath)), - StatusCode::OK); - ASSERT_EQ(manager.getLibrary( - differentOpsLibraryName, - differentOpsLibrary), - StatusCode::OK); - ASSERT_EQ(manager.loadLibrary( - chooseMaxLibraryName, - getGenericFullPathForBazelOut(chooseMaxLibraryPath)), - StatusCode::OK); - ASSERT_EQ(manager.getLibrary( - chooseMaxLibraryName, - chooseMaxLibrary), - StatusCode::OK); - dagDummyModelOutputTensorInfo = std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(pipelineInputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - } -}; - -TEST_F(EnsembleFlowCustomNodeAndDemultiplexerGatherPipelineExecutionTest, MultipleDemultiplexerDummyGathersIntertwinedLevels) { - // Most basic configuration, just process single add-sub custom node pipeline request - // input (differentOps dummy chooseMax ) XN output - // O-----(----->O---------->O------->O------>...----->O - const uint32_t demultiplicationLayersCount = 10; - // values chosen in a way that first chosen different ops result will be addition. all following ones will be multiplications - const std::vector inputValues{0.2, 0.7, -0.4, -0.1, 0.0001, -0.8, 0.7, 0.8, 0.9, 0.1}; - const std::vector inputFactors{1, -1, 2, 2}; - parameters_t parameters{ - {"selection_criteria", "MAXIMUM_MAXIMUM"}}; - // create expected output -> it is dependent from input values & DAG topology - auto expectedResult = inputValues; - std::transform(expectedResult.begin(), expectedResult.end(), expectedResult.begin(), - [demultiplicationLayersCount, inputFactors](float f) { - for (size_t iterations = 0; iterations < demultiplicationLayersCount; ++iterations) { - // input values are prepared in a way that the first layer will choose adding operation tensor - if (iterations == 0) { - f += inputFactors[0]; - } else { - f *= inputFactors[2]; // different ops multiply will be chosen - } - f += 1; // dummy - } - return f; - }); - PredictRequest predictRequest; - this->prepareRequest(predictRequest, inputValues, pipelineInputName); - this->prepareRequest(predictRequest, inputFactors, pipelineFactorsName); - - // create pipeline - std::vector> nodes(2 + 3 * demultiplicationLayersCount); // entry + exit + (choose + differentOps + dummy) * layerCount - const tensor_map_t inputsInfo{{pipelineInputName, dagDummyModelInputTensorInfo}, {pipelineFactorsName, - std::make_shared(pipelineFactorsName, - ovms::Precision::FP32, - ovms::Shape{1, 4}, - Layout{"NC"})}}; - nodes[0] = std::make_unique>(&predictRequest, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, dagDummyModelOutputTensorInfo}}; - nodes[1] = std::make_unique>(&response, outputsInfo); - size_t i = 2; - for (size_t demultiplicationLayer = 0; demultiplicationLayer < demultiplicationLayersCount; ++demultiplicationLayer) { - nodes[i++] = std::make_unique(differentOpsNodeName + "-" + std::to_string(demultiplicationLayer), differentOpsLibrary, parameters_t{}, differentOpsOutputAlias, demultiplyCount); - nodes[i++] = std::make_unique(dummyNodeName + "-" + std::to_string(demultiplicationLayer), "dummy", std::nullopt, modelManager); - nodes[i++] = std::make_unique(chooseMaxNodeName + "-" + std::to_string(demultiplicationLayer), chooseMaxLibrary, parameters, chooseMaxOutputAlias, std::nullopt, std::set({differentOpsNodeName + "-" + std::to_string(demultiplicationLayer)})); - } - - Pipeline pipeline(*nodes[0], *nodes[1], *this->reporter); - i = 2; - for (size_t demultiplicationLayer = 0; demultiplicationLayer < demultiplicationLayersCount; ++demultiplicationLayer) { - if (i == 2) { // first node after entry - pipeline.connect(*nodes[0], *nodes[i], {{pipelineFactorsName, differentOpsFactorsInputName}, {pipelineInputName, differentOpsInputName}}); - } else { // node inside pipeline - pipeline.connect(*nodes[0], *nodes[i], {{pipelineFactorsName, differentOpsFactorsInputName}}); - } - pipeline.connect(*nodes[i], *nodes[i + 1], {{differentOpsOutputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*nodes[i + 1], *nodes[i + 2], {{DUMMY_MODEL_OUTPUT_NAME, chooseMaxInputName}}); - if ((i + 3) != (2 + 3 * demultiplicationLayersCount)) { // connect different ops to choose max - pipeline.connect(*nodes[i + 2], *nodes[i + 3], {{chooseMaxOutputName, differentOpsInputName}}); - } else { // if last connect to exit node - pipeline.connect(*nodes[i + 2], *nodes[1], {{chooseMaxOutputName, pipelineOutputName}}); - } - i = i + 3; - } - for (auto& node : nodes) { - pipeline.push(std::move(node)); - } - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(response.outputs().size(), 1); - this->checkResponse(pipelineOutputName, response, expectedResult, {1, 10}); -} - -TEST_F(EnsembleFlowCustomNodeAndDemultiplexerGatherPipelineExecutionTest, MultipleDemultiplexerLevelsThenDummyThenMultipleGathers) { - // Most basic configuration, just process single add-sub custom node pipeline request - // input (differentOps dummy)xN chooseMax xN output - // O-----(----->O------->O---...----->O---->...----->O - const uint32_t demultiplicationLayersCount = 4; - // values chosen in a way that first chosen different ops result will be addition. all following ones will be multiplications - const std::vector inputValues{0.2, 0.7, -0.4, -0.1, 0.0001, -0.8, 0.7, 0.8, 0.9, 0.1}; - const std::vector inputFactors{1, -1, 2, 2}; - parameters_t parameters{ - {"selection_criteria", "MAXIMUM_MAXIMUM"}}; - // create expected output -> it is dependent from input values & DAG topology - auto expectedResult = inputValues; - std::transform(expectedResult.begin(), expectedResult.end(), expectedResult.begin(), - [demultiplicationLayersCount, inputFactors](float f) { - for (size_t iterations = 0; iterations < demultiplicationLayersCount; ++iterations) { - // input values are prepared in a way that the first layer will choose adding operation tensor - if (iterations == 0) { - f += inputFactors[0]; - } else { - f *= inputFactors[2]; // different ops multiply will be chosen - } - f += 1; // dummy - } - return f; - }); - PredictRequest predictRequest; - this->prepareRequest(predictRequest, inputValues, pipelineInputName); - this->prepareRequest(predictRequest, inputFactors, pipelineFactorsName); - - // create pipeline - size_t nodesCount = 2 + 3 * demultiplicationLayersCount; // entry + exit + (choose + differentOps + dummy) * layerCount - std::vector> nodes(nodesCount); - const tensor_map_t inputsInfo{{pipelineInputName, dagDummyModelInputTensorInfo}, {pipelineFactorsName, - std::make_shared(pipelineFactorsName, - ovms::Precision::FP32, - ovms::Shape{1, 4}, - Layout{"NC"})}}; - nodes[0] = std::make_unique>(&predictRequest, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, dagDummyModelOutputTensorInfo}}; - nodes[nodesCount - 1] = std::make_unique>(&response, outputsInfo); - size_t i = 1; - for (size_t demultiplicationLayer = 0; demultiplicationLayer < demultiplicationLayersCount; ++demultiplicationLayer) { - nodes[i++] = std::make_unique(differentOpsNodeName + "-" + std::to_string(demultiplicationLayer), differentOpsLibrary, parameters_t{}, differentOpsOutputAlias, demultiplyCount); - nodes[i++] = std::make_unique(dummyNodeName + "-" + std::to_string(demultiplicationLayer), "dummy", std::nullopt, modelManager); - nodes[nodesCount - 1 - (i / 2)] = std::make_unique(chooseMaxNodeName + "-" + std::to_string(demultiplicationLayer), chooseMaxLibrary, parameters, chooseMaxOutputAlias, std::nullopt, std::set({differentOpsNodeName + "-" + std::to_string(demultiplicationLayer)})); - } - - Pipeline pipeline(*nodes[0], *nodes[nodesCount - 1], *this->reporter); - i = 1; - for (size_t demultiplicationLayer = 0; demultiplicationLayer < demultiplicationLayersCount; ++demultiplicationLayer) { - if (i == 1) { // first node after entry needs to connect to entry - pipeline.connect(*nodes[0], *nodes[i], {{pipelineFactorsName, differentOpsFactorsInputName}, {pipelineInputName, differentOpsInputName}}); - } - pipeline.connect(*nodes[i], *nodes[i + 1], {{differentOpsOutputName, DUMMY_MODEL_INPUT_NAME}}); - // pass factors further if +2 node is differentOps - if (demultiplicationLayer != demultiplicationLayersCount - 1) { - pipeline.connect(*nodes[i], *nodes[i + 2], {{differentOpsFactorsOutputName, differentOpsFactorsInputName}}); - } - // in between different ops & dummy node - if (demultiplicationLayer != demultiplicationLayersCount - 1) { // all but last dummy connect to differentOps node - pipeline.connect(*nodes[i + 1], *nodes[i + 2], {{DUMMY_MODEL_OUTPUT_NAME, differentOpsInputName}}); - } else { // last dummy connects to chooseMax node - pipeline.connect(*nodes[i + 1], *nodes[i + 2], {{DUMMY_MODEL_OUTPUT_NAME, chooseMaxInputName}}); - } - if (demultiplicationLayer != 0) { // in between choose max nodes - pipeline.connect(*nodes[nodesCount - 1 - (demultiplicationLayer + 1)], - *nodes[nodesCount - 1 - demultiplicationLayer], {{chooseMaxOutputName, chooseMaxInputName}}); - } else { // connect last choose max to exit node - pipeline.connect(*nodes[nodesCount - 1 - (demultiplicationLayer + 1)], - *nodes[nodesCount - 1 - demultiplicationLayer], {{chooseMaxOutputName, pipelineOutputName}}); - } - i = i + 2; - } - for (auto& node : nodes) { - pipeline.push(std::move(node)); - } - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(response.outputs().size(), 1); - this->checkResponse(pipelineOutputName, response, expectedResult, {1, 10}); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, SeriesOfCustomNodes) { - constexpr int N = 100; - constexpr int PARAMETERS_PAIRS_COUNT = 2; - static_assert(PARAMETERS_PAIRS_COUNT > 0); - static_assert(N > PARAMETERS_PAIRS_COUNT); - static_assert((N % PARAMETERS_PAIRS_COUNT) == 0); - // input add-sub x N output - // O------->O->O...O->O------->O - - const std::vector inputValues{3.2, 5.7, -2.4}; - this->prepareRequest(inputValues); - - const std::array addValues{1.5, -2.4}; - const std::array subValues{-5.1, 1.9}; - - auto inputTensorInfo = std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{1, 3}, - Layout{"NC"}); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto tensorInfo = std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{1, 3}, - Layout{"NC"}); - const tensor_map_t outputsInfo{{pipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - std::unique_ptr custom_nodes[N]; - for (int i = 0; i < N; i++) { - custom_nodes[i] = std::make_unique(customNodeName + std::to_string(i), library, - parameters_t{ - {"add_value", std::to_string(addValues[i % PARAMETERS_PAIRS_COUNT])}, - {"sub_value", std::to_string(subValues[i % PARAMETERS_PAIRS_COUNT])}}); - } - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *(custom_nodes[0]), {{pipelineInputName, customNodeInputName}}); - pipeline.connect(*(custom_nodes[N - 1]), *output_node, {{customNodeOutputName, pipelineOutputName}}); - for (int i = 0; i < N - 1; i++) { - pipeline.connect(*(custom_nodes[i]), *(custom_nodes[i + 1]), {{customNodeOutputName, customNodeInputName}}); - } - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(output_node)); - for (auto& custom_node : custom_nodes) { - pipeline.push(std::move(custom_node)); - } - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(response.outputs().size(), 1); - - this->checkResponse(inputValues, [N, addValues, subValues, PARAMETERS_PAIRS_COUNT](float value) -> float { - for (int i = 0; i < PARAMETERS_PAIRS_COUNT; i++) { - value += (N / PARAMETERS_PAIRS_COUNT) * addValues[i]; - value -= (N / PARAMETERS_PAIRS_COUNT) * subValues[i]; - } - return value; - }); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, ParallelCustomNodes) { - constexpr int N = 200; - constexpr int PARAMETERS_PAIRS_COUNT = 5; - static_assert(PARAMETERS_PAIRS_COUNT > 0); - static_assert(N > PARAMETERS_PAIRS_COUNT); - static_assert((N % PARAMETERS_PAIRS_COUNT) == 0); - /* input add-sub x N output - O---------->O------------->O - ... ... /\ - L---------->O-------------_| - */ - - const std::vector inputValues{9.1, -3.7, 22.2}; - this->prepareRequest(inputValues); - - const std::array addValues{4.5, 0.2, -0.6, 0.4, -2.5}; - const std::array subValues{8.5, -3.2, 10.0, -0.5, 2.4}; - - auto inputTensorInfo = std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{1, 3}, - Layout{"NC"}); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - tensor_map_t outputsInfo; - for (size_t i = 0; i < N; ++i) { - const std::string outputName = pipelineOutputName + std::to_string(i); - outputsInfo.emplace(outputName, - std::make_shared(outputName, - ovms::Precision::FP32, - ovms::Shape{1, 3}, - Layout{"NC"})); - } - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - std::unique_ptr custom_nodes[N]; - for (int i = 0; i < N; i++) { - custom_nodes[i] = std::make_unique(customNodeName + std::to_string(i), library, - parameters_t{ - {"add_value", std::to_string(addValues[i % PARAMETERS_PAIRS_COUNT])}, - {"sub_value", std::to_string(subValues[i % PARAMETERS_PAIRS_COUNT])}}); - pipeline.connect(*input_node, *(custom_nodes[i]), - {{pipelineInputName, customNodeInputName}}); - pipeline.connect(*(custom_nodes[i]), *output_node, - {{customNodeOutputName, pipelineOutputName + std::to_string(i)}}); - pipeline.push(std::move(custom_nodes[i])); - } - pipeline.push(std::move(input_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(response.outputs().size(), N); - - for (int i = 0; i < N; i++) { - this->checkResponse( - pipelineOutputName + std::to_string(i), - inputValues, - [i, addValues, subValues, PARAMETERS_PAIRS_COUNT](float value) -> float { - value += addValues[i % PARAMETERS_PAIRS_COUNT]; - value -= subValues[i % PARAMETERS_PAIRS_COUNT]; - return value; - }); - } -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, CustomAndDLNodes) { - // input add-sub1 dummy add-sub2 output - // O------->O------O--------O------>O - ConstructorEnabledModelManager modelManager; - ModelConfig config = DUMMY_MODEL_CONFIG; - modelManager.reloadModelWithVersions(config); - - const std::vector inputValues{ - 4, 1.5, -5, -2.5, 9.3, 0.3, -0.15, 7.4, 5.2, -2.4}; - this->prepareRequest(inputValues); - - const float addValues[] = {-0.85, 30.2}; - const float subValues[] = {1.35, -28.5}; - - const tensor_map_t inputsInfo{{pipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - auto model_node = std::make_unique( - "dummy_node", - "dummy", - std::nullopt, - modelManager); - std::unique_ptr custom_node[] = { - std::make_unique(customNodeName + "_0", library, - parameters_t{ - {"add_value", std::to_string(addValues[0])}, - {"sub_value", std::to_string(subValues[0])}}), - std::make_unique(customNodeName + "_1", library, - parameters_t{ - {"add_value", std::to_string(addValues[1])}, - {"sub_value", std::to_string(subValues[1])}})}; - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *(custom_node[0]), {{pipelineInputName, customNodeInputName}}); - pipeline.connect(*(custom_node[0]), *model_node, {{customNodeOutputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *(custom_node[1]), {{DUMMY_MODEL_OUTPUT_NAME, customNodeInputName}}); - pipeline.connect(*(custom_node[1]), *output_node, {{customNodeOutputName, pipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(custom_node[0])); - pipeline.push(std::move(custom_node[1])); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(response.outputs().size(), 1); - - this->checkResponse(inputValues, [addValues, subValues](float value) -> float { - return value + DUMMY_ADDITION_VALUE + addValues[0] + addValues[1] - subValues[0] - subValues[1]; - }); -} - -struct LibraryWithScalarOutput { - static constexpr float libraryScalarNodeAddValue = 2.1f; - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** handle, int* outputsNum, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - if (inputsCount != 1) - return 1; - *handle = (struct CustomNodeTensor*)malloc(sizeof(struct CustomNodeTensor)); - *outputsNum = 1; - (*handle)->name = "output_numbers"; - (*handle)->precision = CustomNodeTensorPrecision::FP32; - (*handle)->dims = (uint64_t*)malloc(0); - (*handle)->dimsCount = 0; - (*handle)->data = (uint8_t*)malloc(sizeof(float)); - *((float*)(*handle)->data) = *((float*)inputs[0].data) + libraryScalarNodeAddValue; - (*handle)->dataBytes = sizeof(float); - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, CustomNodeScalarsAndScalarDLNodeInTheMiddle) { - // input CN DL CN output - // O------->O------->O--------O------>O - // +const passthru +const - ConstructorEnabledModelManager modelManager; - ModelConfig config = SCALAR_MODEL_CONFIG; - modelManager.reloadModelWithVersions(config); - - const std::vector inputValues{5.4f}; - const std::vector expectedOutputValues{inputValues[0] + LibraryWithScalarOutput::libraryScalarNodeAddValue * 2}; - this->prepareRequest(this->request, inputValues, pipelineInputName, {}); - ((*this->request.mutable_inputs())[pipelineInputName]).mutable_tensor_shape()->Clear(); - - const tensor_map_t inputsInfo{{pipelineInputName, std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{}, - Layout{"..."})}}; - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{}, - Layout{"..."})}}; - auto output_node = std::make_unique>(&response, outputsInfo); - auto model_node = std::make_unique( - "scalar_node", - "scalar", - std::nullopt, - modelManager); - std::unique_ptr custom_node[] = { - std::make_unique("scalar_node_0", createLibraryMock(), - parameters_t{}), - std::make_unique("scalar_node_1", createLibraryMock(), - parameters_t{})}; - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *(custom_node[0]), {{pipelineInputName, "anything"}}); - pipeline.connect(*(custom_node[0]), *model_node, {{"output_numbers", SCALAR_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *(custom_node[1]), {{SCALAR_MODEL_OUTPUT_NAME, "anything"}}); - pipeline.connect(*(custom_node[1]), *output_node, {{"output_numbers", pipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(custom_node[0])); - pipeline.push(std::move(custom_node[1])); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(this->response.mutable_outputs()->count(pipelineOutputName), 1); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_shape()->dim_size(), 0); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_content()->size(), sizeof(float)); - ASSERT_EQ(*(float*)(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_content()->data()), expectedOutputValues[0]); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, DLNodeScalarsAndScalarCustomNodeInTheMiddle) { - // input DL CN DL output - // O------->O------->O--------O------>O - // passthru +const passthru - ConstructorEnabledModelManager modelManager; - ModelConfig config = SCALAR_MODEL_CONFIG; - modelManager.reloadModelWithVersions(config); - - const std::vector inputValues{5.4f}; - const std::vector expectedOutputValues{inputValues[0] + LibraryWithScalarOutput::libraryScalarNodeAddValue}; - this->prepareRequest(this->request, inputValues, pipelineInputName, {}); - ((*this->request.mutable_inputs())[pipelineInputName]).mutable_tensor_shape()->Clear(); - - const tensor_map_t inputsInfo{{pipelineInputName, std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{}, - Layout{"..."})}}; - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{}, - Layout{"..."})}}; - auto output_node = std::make_unique>(&response, outputsInfo); - auto custom_node = std::make_unique("scalar_node_0", createLibraryMock(), - parameters_t{}); - std::unique_ptr model_node[] = { - std::make_unique("scalar_model_0", "scalar", std::nullopt, modelManager), - std::make_unique("scalar_model_1", "scalar", std::nullopt, modelManager)}; - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *(model_node[0]), {{pipelineInputName, SCALAR_MODEL_INPUT_NAME}}); - pipeline.connect(*(model_node[0]), *custom_node, {{SCALAR_MODEL_OUTPUT_NAME, "anything"}}); - pipeline.connect(*custom_node, *(model_node[1]), {{"output_numbers", SCALAR_MODEL_INPUT_NAME}}); - pipeline.connect(*(model_node[1]), *output_node, {{SCALAR_MODEL_OUTPUT_NAME, pipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node[0])); - pipeline.push(std::move(model_node[1])); - pipeline.push(std::move(custom_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(this->response.mutable_outputs()->count(pipelineOutputName), 1); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_shape()->dim_size(), 0); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_content()->size(), sizeof(float)); - ASSERT_EQ(*(float*)(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_content()->data()), expectedOutputValues[0]); -} - -struct LibraryFailInExecute { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor*, int, struct CustomNodeTensor**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 1; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FailInCustomNodeExecution) { - auto pipeline = this->prepareSingleNodePipelineWithLibraryMock(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_EXECUTION_FAILED); -} - -struct LibraryCorruptedOutputHandle { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor*, int, struct CustomNodeTensor** handle, int* outputsNum, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - *handle = nullptr; - *outputsNum = 5; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FailInCustomNodeOutputsCorruptedHandle) { - auto pipeline = this->prepareSingleNodePipelineWithLibraryMock(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED); -} - -struct LibraryCorruptedOutputsNumber { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor*, int, struct CustomNodeTensor** handle, int* outputsNum, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - *handle = (struct CustomNodeTensor*)malloc(5 * sizeof(struct CustomNodeTensor)); - *outputsNum = 0; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FailInCustomNodeOutputsCorruptedNumberOfOutputs) { - auto pipeline = this->prepareSingleNodePipelineWithLibraryMock(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED_COUNT); -} - -struct LibraryMissingOutput { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor*, int, struct CustomNodeTensor** handle, int* outputsNum, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - *handle = (struct CustomNodeTensor*)malloc(sizeof(struct CustomNodeTensor)); - *outputsNum = 1; - (*handle)->name = "random_not_connected_output"; - (*handle)->precision = CustomNodeTensorPrecision::FP32; - (*handle)->dims = (uint64_t*)malloc(sizeof(uint64_t)); - (*handle)->dims[0] = 1; - (*handle)->dimsCount = 1; - (*handle)->data = (uint8_t*)malloc(sizeof(float) * sizeof(uint8_t)); - (*handle)->dataBytes = sizeof(float); - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FailInCustomNodeMissingOutput) { - auto pipeline = this->prepareSingleNodePipelineWithLibraryMock(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_MISSING_OUTPUT); -} - -struct LibraryIncorrectOutputPrecision { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor*, int, struct CustomNodeTensor** handle, int* outputsNum, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - *handle = (struct CustomNodeTensor*)malloc(sizeof(struct CustomNodeTensor)); - *outputsNum = 1; - (*handle)->name = "output_numbers"; - (*handle)->precision = CustomNodeTensorPrecision::UNSPECIFIED; - (*handle)->dims = (uint64_t*)malloc(sizeof(uint64_t)); - (*handle)->dimsCount = 1; - (*handle)->data = (uint8_t*)malloc(sizeof(uint8_t)); - (*handle)->dataBytes = 1; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FailInCustomNodeOutputInvalidPrecision) { - auto pipeline = this->prepareSingleNodePipelineWithLibraryMock(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_INVALID_PRECISION); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, SuccessLibraryWithScalarOutput) { - const std::vector inputValues{3.5f}; - const std::vector expectedOutputValues{inputValues[0] + LibraryWithScalarOutput::libraryScalarNodeAddValue}; - auto inputTensorInfo = std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{}, - Layout{"..."}); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - this->prepareRequest(this->request, inputValues, pipelineInputName, {}); - ((*this->request.mutable_inputs())[pipelineInputName]).mutable_tensor_shape()->Clear(); - auto input_node = std::make_unique>(&request, inputsInfo); - auto outputTensorInfo = std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{}, - Layout{"..."}); - const tensor_map_t outputsInfo{{pipelineOutputName, outputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - auto custom_node = std::make_unique( - customNodeName, - createLibraryMock(), - parameters_t{}); - - auto pipeline = std::make_unique(*input_node, *output_node, *this->reporter); - pipeline->connect(*input_node, *custom_node, {{pipelineInputName, customNodeInputName}}); - pipeline->connect(*custom_node, *output_node, {{customNodeOutputName, pipelineOutputName}}); - - pipeline->push(std::move(input_node)); - pipeline->push(std::move(custom_node)); - pipeline->push(std::move(output_node)); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(this->response.mutable_outputs()->count(pipelineOutputName), 1); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_shape()->dim_size(), 0); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_content()->size(), sizeof(float)); - ASSERT_EQ(*(float*)(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_content()->data()), expectedOutputValues[0]); -} - -struct LibraryIncorrectOutputContentSize { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor*, int, struct CustomNodeTensor** handle, int* outputsNum, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - *handle = (struct CustomNodeTensor*)malloc(sizeof(struct CustomNodeTensor)); - *outputsNum = 1; - (*handle)->name = "output_numbers"; - (*handle)->precision = CustomNodeTensorPrecision::FP32; - (*handle)->dims = (uint64_t*)malloc(sizeof(uint64_t)); - (*handle)->dimsCount = 1; - (*handle)->data = nullptr; - (*handle)->dataBytes = 0; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FailInCustomNodeOutputInvalidContentSize) { - auto pipeline = this->prepareSingleNodePipelineWithLibraryMock(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_INVALID_CONTENT_SIZE); -} - -struct LibraryNotInitilizedExecuteCorrectly { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - // execute function is not using buffer allocation feature, therefore initialize does not do anything apart from returning 0 meaning that initialize worked as intended - static int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - *outputs = (struct CustomNodeTensor*)malloc(sizeof(struct CustomNodeTensor)); - *outputsCount = 1; - (*outputs)->name = "output_numbers"; - (*outputs)->precision = CustomNodeTensorPrecision::FP32; - (*outputs)->dims = (uint64_t*)malloc(2 * sizeof(uint64_t)); - (*outputs)->dims[0] = 1; - (*outputs)->dims[1] = 10; - (*outputs)->dimsCount = 2; - (*outputs)->data = (uint8_t*)malloc(sizeof(uint8_t)); - (*outputs)->dataBytes = 40; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, SuccessInCustomNodeExecutionNotInitialized) { - auto pipeline = this->prepareSingleNodePipelineWithLibraryMock(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); -} - -struct LibraryNotInitializedFailInExecute { - // execute is using buffer allocation, therefore initialize should be modified to work properly - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - // execute function is using customNodeLibraryInternalManager, that was supposed to be created in initialize function - // execute fails due to incorrect initialization - static int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - if (customNodeLibraryInternalManager == nullptr) { - return 1; - } - *outputs = (struct CustomNodeTensor*)malloc(sizeof(struct CustomNodeTensor)); - *outputsCount = 1; - (*outputs)->name = "output_numbers"; - (*outputs)->precision = CustomNodeTensorPrecision::FP32; - (*outputs)->dims = (uint64_t*)malloc(2 * sizeof(uint64_t)); - (*outputs)->dims[0] = 1; - (*outputs)->dims[1] = 10; - (*outputs)->dimsCount = 2; - (*outputs)->data = (uint8_t*)malloc(sizeof(uint8_t)); - (*outputs)->dataBytes = 40; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FailInCustomNodeExecutionNotInitialized) { - auto pipeline = this->prepareSingleNodePipelineWithLibraryMock(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_EXECUTION_FAILED); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FailInCustomNodeInitialize) { - // Nodes - // request custom response - // O--------->O---------->O - // add-sub - ConstructorEnabledModelManager manager; - PipelineFactory factory; - - const float addValue = 0.9; - const float subValue = 7.3; - - // initialize function call from now on will be calling this lambda function, which indicates - // initialization failure - library.initialize = [](void**, const struct CustomNodeParam*, int) { return 1; }; - ASSERT_TRUE(library.isValid()); - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, library, parameters_t{{"add_value", std::to_string(addValue)}, {"sub_value", std::to_string(subValue)}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (pipelineInputName) O--------->O custom node (customNodeInputName) - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, customNodeInputName}}}}; - - // custom node (customNodeOutputName) O--------->O response (pipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"custom_node", {{customNodeOutputName, pipelineOutputName}}}}; - - // createDefinition fails due to initialization failure - ASSERT_EQ(factory.createDefinition("my_new_pipeline", info, connections, manager, manager, manager), StatusCode::NODE_LIBRARY_INITIALIZE_FAILED); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FailInCustomNodeDeinitialize) { - // Nodes - // request custom response - // O--------->O---------->O - // add-sub - ConstructorEnabledModelManager manager; - PipelineFactory factory; - - const std::vector inputValues{7.8, -2.4, 1.9, 8.7, -2.4, 3.5}; - this->prepareRequest(inputValues); - - const float addValue = 0.9; - const float subValue = 7.3; - - // deinitialize function call from now on will be calling this lambda function, which indicates - // deinitialization failure - library.deinitialize = [](void*) { return 1; }; - ASSERT_TRUE(library.isValid()); - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, library, parameters_t{{"add_value", std::to_string(addValue)}, {"sub_value", std::to_string(subValue)}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (pipelineInputName) O--------->O custom node (customNodeInputName) - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, customNodeInputName}}}}; - - // custom node (customNodeOutputName) O--------->O response (pipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"custom_node", {{customNodeOutputName, pipelineOutputName}}}}; - - std::unique_ptr pipeline; - // creating definition, pipeline and then executing works properly due to correct initialization - ASSERT_EQ(factory.createDefinition("my_new_pipeline", info, connections, manager, manager, manager), StatusCode::OK); - ASSERT_EQ(factory.create(pipeline, "my_new_pipeline", &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - this->checkResponse(inputValues, [addValue, subValue](float value) -> float { - return value + addValue - subValue; - }); - - // after execute we are retiring pipeline definition and making sure that its state is retired after the operation - // even tho deinitialize is failing there is no direct indication of that apart from errors in logs - factory.retireOtherThan({}, manager); - ASSERT_EQ(factory.findDefinitionByName("my_new_pipeline")->getStateCode(), PipelineDefinitionStateCode::RETIRED); -} - -class EnsembleFlowCustomNodeFactoryCreateThenExecuteTest : public EnsembleFlowCustomNodePipelineExecutionTest {}; - -TEST_F(EnsembleFlowCustomNodeFactoryCreateThenExecuteTest, SimplePipelineFactoryCreationWithCustomNode) { - // Nodes - // request custom response - // O--------->O---------->O - // add-sub - ConstructorEnabledModelManager manager; - PipelineFactory factory; - - const std::vector inputValues{7.8, -2.4, 1.9, 8.7, -2.4, 3.5}; - this->prepareRequest(inputValues); - - const float addValue = 0.9; - const float subValue = 7.3; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, library, parameters_t{{"add_value", std::to_string(addValue)}, {"sub_value", std::to_string(subValue)}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (pipelineInputName) O--------->O custom node (customNodeInputName) - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, customNodeInputName}}}}; - - // custom node (customNodeOutputName) O--------->O response (pipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"custom_node", {{customNodeOutputName, pipelineOutputName}}}}; - - std::unique_ptr pipeline; - ASSERT_EQ(factory.createDefinition("my_new_pipeline", info, connections, manager, manager, manager), StatusCode::OK); - ASSERT_EQ(factory.create(pipeline, "my_new_pipeline", &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - this->checkResponse(inputValues, [addValue, subValue](float value) -> float { - return value + addValue - subValue; - }); -} - -TEST_F(EnsembleFlowCustomNodeFactoryCreateThenExecuteTest, ParallelPipelineFactoryUsageWithCustomNode) { - // Nodes - // custom_node_N - // v-------->O----------v - // request O--------->O---------->O response x PARALLEL_SIMULATED_REQUEST_COUNT - // ^-------->O----------^ - // add-sub - ConstructorEnabledModelManager manager; - PipelineFactory factory; - - const int PARALLEL_CUSTOM_NODES = 3; - const int PARALLEL_SIMULATED_REQUEST_COUNT = 30; - - const std::vector inputValues{7.8, -2.4, 1.9, 8.7, -2.4, 3.5}; - std::array requests{}; - - for (int i = 0; i < PARALLEL_SIMULATED_REQUEST_COUNT; i++) { - this->prepareRequest(requests[i], inputValues); - } - - const std::array addValues{-1.5, 1.4, -0.1}; - const std::array subValues{4.9, -1.9, -0.9}; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - for (int i = 0; i < PARALLEL_CUSTOM_NODES; i++) { - info.emplace_back(std::move(NodeInfo( - NodeKind::CUSTOM, - "custom_node_" + std::to_string(i), - "", std::nullopt, - {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, - library, parameters_t{{"add_value", std::to_string(addValues[i])}, {"sub_value", std::to_string(subValues[i])}}))); - } - - pipeline_connections_t connections; - - for (int i = 0; i < PARALLEL_CUSTOM_NODES; i++) { - // request (pipelineInputName) O--------->O custom_node_N (customNodeInputName) - connections["custom_node_" + std::to_string(i)] = { - {ENTRY_NODE_NAME, {{pipelineInputName, customNodeInputName}}}}; - } - - auto& responseConnections = connections[EXIT_NODE_NAME]; - for (int i = 0; i < PARALLEL_CUSTOM_NODES; i++) { - responseConnections["custom_node_" + std::to_string(i)] = - {{customNodeOutputName, "output_" + std::to_string(i)}}; - } - - std::unique_ptr pipeline; - ASSERT_EQ(factory.createDefinition("my_new_pipeline", info, connections, manager, manager, manager), StatusCode::OK); - ASSERT_EQ(factory.create(pipeline, "my_new_pipeline", &requests[0], &response, manager), StatusCode::OK); - - auto run = [this, &requests, &manager, &factory, &inputValues, addValues, subValues, PARALLEL_CUSTOM_NODES](int i) { - std::unique_ptr pipeline; - PredictResponse response_local; - - ASSERT_EQ(factory.create(pipeline, "my_new_pipeline", &requests[i], &response_local, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - for (int n = 0; n < PARALLEL_CUSTOM_NODES; n++) { - this->checkResponse("output_" + std::to_string(n), response_local, inputValues, [addValues, subValues, n](float value) -> float { - return value + addValues[n] - subValues[n]; - }); - } - }; - - std::vector> promises(PARALLEL_SIMULATED_REQUEST_COUNT); - std::vector threads; - - for (int n = 0; n < PARALLEL_SIMULATED_REQUEST_COUNT; n++) { - threads.emplace_back(std::thread([&promises, n, &run]() { - promises[n].get_future().get(); - run(n); - })); - } - - // Sleep to allow all threads to initialize - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - - for (auto& promise : promises) { - promise.set_value(); - } - - for (auto& thread : threads) { - thread.join(); - } -} - -struct AddSubInternalManager { - uint64_t* inputDims; - uint64_t* outputDims; - struct CustomNodeTensorInfo* inputInfo; - struct CustomNodeTensorInfo* outputInfo; - struct CustomNodeTensor* outputTensor; - uint8_t* outputTensorData; - uint64_t* outputTensorDims; - inline static std::vector mockedOutput{0, 0, -1, 1, -2, 2, -3, 3, -4, 4}; - - AddSubInternalManager() { - inputDims = (uint64_t*)malloc(2 * sizeof(uint64_t)); - outputDims = (uint64_t*)malloc(2 * sizeof(uint64_t)); - inputInfo = (struct CustomNodeTensorInfo*)malloc(1 * sizeof(struct CustomNodeTensorInfo)); - outputInfo = (struct CustomNodeTensorInfo*)malloc(1 * sizeof(struct CustomNodeTensorInfo)); - outputTensor = (struct CustomNodeTensor*)malloc(1 * sizeof(struct CustomNodeTensor)); - outputTensorData = (uint8_t*)malloc(10 * 4 * sizeof(uint8_t)); - outputTensorDims = (uint64_t*)malloc(2 * sizeof(uint64_t)); - } - - bool isPtrOwnedByManager(void* ptr) { - return (ptr == inputDims || ptr == outputDims || ptr == inputInfo || ptr == outputInfo || ptr == outputTensor || ptr == outputTensorData || ptr == outputTensorDims); - } -}; - -struct LibraryAddSubWithInternalManager { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - AddSubInternalManager* internalManager = new AddSubInternalManager(); - *customNodeLibraryInternalManager = internalManager; - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - if (customNodeLibraryInternalManager != nullptr) { - AddSubInternalManager* internalManager = static_cast(customNodeLibraryInternalManager); - delete internalManager; - } - return 0; - } - static int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - AddSubInternalManager* internalManager = static_cast(customNodeLibraryInternalManager); - if (internalManager == nullptr) - return 1; - - const struct CustomNodeTensor* input = &inputs[0]; - - *outputsCount = 1; - *outputs = internalManager->outputTensor; - struct CustomNodeTensor* output = (&(*outputs))[0]; - - output->name = "output_numbers"; - output->data = internalManager->outputTensorData; - output->dataBytes = input->dataBytes; - output->dims = internalManager->outputTensorDims; - output->dimsCount = input->dimsCount; - memcpy((void*)output->dims, (void*)input->dims, input->dimsCount * sizeof(uint64_t)); - output->precision = input->precision; - - for (uint64_t i = 0; i < output->dataBytes; i += sizeof(float)) { - *(float*)(output->data + i) = internalManager->mockedOutput[i / sizeof(float)]; - } - - return 0; - } - - static int getInputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - AddSubInternalManager* internalManager = static_cast(customNodeLibraryInternalManager); - if (internalManager == nullptr) - return 1; - *infoCount = 1; - *info = internalManager->inputInfo; - (*info)->name = "input_numbers"; - (*info)->dimsCount = 2; - (*info)->dims = internalManager->inputDims; - (*info)->dims[0] = 1; - (*info)->dims[1] = 10; - (*info)->precision = FP32; - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - AddSubInternalManager* internalManager = static_cast(customNodeLibraryInternalManager); - if (internalManager == nullptr) - return 1; - *infoCount = 1; - *info = internalManager->outputInfo; - (*info)->name = "output_numbers"; - (*info)->dimsCount = 2; - (*info)->dims = internalManager->outputDims; - (*info)->dims[0] = 1; - (*info)->dims[1] = 10; - (*info)->precision = FP32; - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - AddSubInternalManager* internalManager = static_cast(customNodeLibraryInternalManager); - if (internalManager == nullptr) - return 1; - if (!internalManager->isPtrOwnedByManager(ptr)) { - free(ptr); - } - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodeFactoryCreateThenExecuteTest, PipelineFactoryCreationAndExecuteWithCustomNodeUsingInternalManager) { - ConstructorEnabledModelManager manager; - PipelineFactory factory; - - const std::vector inputValues{7.8, -2.4, 1.9, 8.7, -2.4, 3.5, 2.5, 1.2, -2.5, 10.0}; - this->prepareRequest(inputValues); - - NodeLibrary libraryAddSubWithInternalManager = createLibraryMock(); - ASSERT_TRUE(libraryAddSubWithInternalManager.isValid()); - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, libraryAddSubWithInternalManager}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (pipelineInputName) O--------->O custom node (customNodeInputName) - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, customNodeInputName}}}}; - - // custom node (customNodeOutputName) O--------->O response (pipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"custom_node", {{customNodeOutputName, pipelineOutputName}}}}; - - std::unique_ptr pipeline; - ASSERT_EQ(factory.createDefinition("my_new_pipeline", info, connections, manager, manager, manager), StatusCode::OK); - ASSERT_EQ(factory.create(pipeline, "my_new_pipeline", &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - this->checkResponse(AddSubInternalManager::mockedOutput, [](float value) -> float { - return value; - }); -} - -static const char* pipelineCustomNodeConfig = R"( -{ - "model_config_list": [], - "custom_node_library_config_list": [ - { - "name": "lib_add_sub", - "base_path": "/ovms/bazel-bin/src/lib_node_add_sub.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_add_sub", - "params": { - "add_value": "3.2", - "sub_value": "2.7" - }, - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -class EnsembleFlowCustomNodeLoadConfigThenExecuteTest : public EnsembleFlowCustomNodePipelineExecutionTest { -protected: - void SetUp() override { - EnsembleFlowCustomNodePipelineExecutionTest::SetUp(); - configJsonFilePath = directoryPath + "/ovms_config_file.json"; - } - - void loadCorrectConfiguration() { - this->loadConfiguration(pipelineCustomNodeConfig); - } - - void loadConfiguration(const char* configContent, Status expectedStatus = StatusCode::OK) { - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(configContent), configJsonFilePath); - ASSERT_EQ(manager.loadConfig(configJsonFilePath), expectedStatus); - } - - void checkResponseForCorrectConfiguration() { - this->checkResponse(inputValues, [](float value) -> float { - return value + 3.2 - 2.7; - }); - } - - std::string configJsonFilePath; - const std::string pipelineName = "my_pipeline"; - ConstructorEnabledModelManager manager; - const std::vector inputValues{2.4, 9.3, -7.1}; -}; - -TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, AddSubCustomNode) { - std::unique_ptr pipeline; - this->prepareRequest(inputValues); - this->loadCorrectConfiguration(); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponseForCorrectConfiguration(); -} - -static const char* pipelineCustomNodeReferenceMissingLibraryConfig = R"( -{ - "model_config_list": [], - "custom_node_library_config_list": [ - { - "name": "lib_add_sub", - "base_path": "/ovms/bazel-bin/src/lib_node_add_sub.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "custom_node", - "library_name": "non_existing_library", - "params": { - "add_value": "3.2", - "sub_value": "2.7" - }, - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, ReferenceMissingLibraryThenCorrect) { - std::unique_ptr pipeline; - this->prepareRequest(inputValues); - - // Loading correct configuration is required for test to pass. - // This is due to fact that when OVMS loads pipeline definition for the first time and fails, its status is RETIRED. - this->loadCorrectConfiguration(); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponseForCorrectConfiguration(); - response.Clear(); - - this->loadConfiguration(pipelineCustomNodeReferenceMissingLibraryConfig, StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET); - response.Clear(); - - this->loadCorrectConfiguration(); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponseForCorrectConfiguration(); -} - -static const char* pipelineCustomNodeReferenceLibraryWithExecutionErrorMissingParamsLibraryConfig = R"( -{ - "model_config_list": [], - "custom_node_library_config_list": [ - { - "name": "lib_add_sub_new", - "base_path": "/ovms/bazel-bin/src/lib_node_add_sub.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_add_sub_new", - "params": { - }, - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, ReferenceLibraryWithExecutionErrorThenCorrect) { - std::unique_ptr pipeline; - this->prepareRequest(inputValues); - - // Loading correct configuration is required for test to pass. - // This is due to fact that when OVMS loads pipeline definition for the first time and fails, its status is RETIRED. - this->loadCorrectConfiguration(); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponseForCorrectConfiguration(); - response.Clear(); - - this->loadConfiguration(pipelineCustomNodeReferenceLibraryWithExecutionErrorMissingParamsLibraryConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_EXECUTION_FAILED); - response.Clear(); - - this->loadCorrectConfiguration(); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponseForCorrectConfiguration(); -} - -static const char* pipelineCustomNodeMissingParametersConfig = R"( -{ - "model_config_list": [], - "custom_node_library_config_list": [ - { - "name": "lib_add_sub", - "base_path": "/ovms/bazel-bin/src/lib_node_add_sub.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_add_sub", - "params": { - "random_parameter": "abcd" - }, - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, MissingRequiredNodeParametersThenCorrect) { - std::unique_ptr pipeline; - this->prepareRequest(inputValues); - - // Loading correct configuration is required for test to pass. - // This is due to fact that when OVMS loads pipeline definition for the first time and fails, its status is RETIRED. - this->loadCorrectConfiguration(); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponseForCorrectConfiguration(); - response.Clear(); - - this->loadConfiguration(pipelineCustomNodeMissingParametersConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_EXECUTION_FAILED); - response.Clear(); - - this->loadCorrectConfiguration(); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponseForCorrectConfiguration(); -} - -static const char* pipelineCustomNodeLibraryNotEscapedPathConfig = R"( -{ - "model_config_list": [], - "custom_node_library_config_list": [ - { - "name": "lib_add_sub_new", - "base_path": "/ovms/bazel-bin/src/../src/lib_node_add_sub.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_add_sub_new", - "params": { - "add_value": "3.2", - "sub_value": "2.7" - }, - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, ReferenceLibraryWithRestrictedBasePathThenCorrect) { - std::unique_ptr pipeline; - this->prepareRequest(inputValues); - - // Loading correct configuration is required for test to pass. - // This is due to fact that when OVMS loads pipeline definition for the first time and fails, its status is RETIRED. - this->loadCorrectConfiguration(); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponseForCorrectConfiguration(); - response.Clear(); - - this->loadConfiguration(pipelineCustomNodeLibraryNotEscapedPathConfig, StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET); - response.Clear(); - - this->loadCorrectConfiguration(); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponseForCorrectConfiguration(); -} - -static const char* pipelineCustomNodeDifferentOperationsConfig = R"( -{ - "model_config_list": [], - "custom_node_library_config_list": [ - { - "name": "lib_perform_different_operations", - "base_path": "/ovms/bazel-bin/src/lib_node_perform_different_operations.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input", "pipeline_factors"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_perform_different_operations", - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}}, - {"op_factors": {"node_name": "request", - "data_item": "pipeline_factors"}} - ], - "outputs": [ - {"data_item": "different_ops_results", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -class EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest : public EnsembleFlowCustomNodeLoadConfigThenExecuteTest { -protected: - void SetUp() override { - EnsembleFlowCustomNodeLoadConfigThenExecuteTest::SetUp(); - configJsonFilePath = directoryPath + "/ovms_config_file.json"; - } - const std::string differentOpsInputName = "pipeline_input"; - const std::string differentOpsFactorsName = "pipeline_factors"; -}; - -enum OPS { - ADD, - SUB, - MULTIPLY, - DIVIDE -}; - -static void prepareDifferentOpsExpectedOutput(std::vector& expectedOutput, const std::vector& input, const std::vector& factors) { - for (size_t j = 0; j < 4; ++j) { // iterate over ops - for (size_t i = 0; i < DUMMY_MODEL_OUTPUT_SIZE; ++i) { - size_t index = DUMMY_MODEL_OUTPUT_SIZE * j + i; - switch (j) { - case ADD: - expectedOutput[index] = input[i] + factors[j]; - break; - case SUB: - expectedOutput[index] = input[i] - factors[j]; - break; - case MULTIPLY: - expectedOutput[index] = input[i] * factors[j]; - break; - case DIVIDE: - expectedOutput[index] = input[i] / factors[j]; - break; - } - } - } -} - -enum class Method { - MAXIMUM_MAXIMUM, - MAXIMUM_MINIMUM, - MAXIMUM_AVERAGE, -}; - -static std::vector prepareGatherHighestExpectedOutput(std::vector input, Method option) { - std::vector expectedOutput(DUMMY_MODEL_OUTPUT_SIZE); - size_t tensorsCount = input.size() / DUMMY_MODEL_OUTPUT_SIZE; - // perform operations - std::vector minimums(tensorsCount, std::numeric_limits::max()); - std::vector maximums(tensorsCount, std::numeric_limits::lowest()); - std::vector averages(tensorsCount, 0); - for (size_t opId = 0; opId < tensorsCount; ++opId) { // iterate over ops - for (size_t i = 0; i < DUMMY_MODEL_OUTPUT_SIZE; ++i) { - size_t index = DUMMY_MODEL_OUTPUT_SIZE * opId + i; - switch (option) { - case Method::MAXIMUM_MAXIMUM: - maximums[opId] = std::max(maximums[opId], input[index]); - break; - case Method::MAXIMUM_MINIMUM: - minimums[opId] = std::min(maximums[opId], input[index]); - break; - case Method::MAXIMUM_AVERAGE: - averages[opId] += input[index]; - break; - default: - throw std::logic_error(""); - break; - } - } - averages[opId] /= DUMMY_MODEL_OUTPUT_SIZE; - } - // choose tensor - size_t whichTensor = 42; - const std::vector* fromWhichContainerToChoose = &maximums; - switch (option) { - case Method::MAXIMUM_MAXIMUM: - fromWhichContainerToChoose = &maximums; - break; - case Method::MAXIMUM_MINIMUM: - fromWhichContainerToChoose = &minimums; - break; - case Method::MAXIMUM_AVERAGE: - fromWhichContainerToChoose = &averages; - break; - default: - throw std::logic_error(""); - } - whichTensor = std::distance(fromWhichContainerToChoose->begin(), - std::max_element(fromWhichContainerToChoose->begin(), - fromWhichContainerToChoose->end())); - // copy tensor - std::copy(input.begin() + DUMMY_MODEL_OUTPUT_SIZE * whichTensor, - input.begin() + DUMMY_MODEL_OUTPUT_SIZE * (whichTensor + 1), - expectedOutput.begin()); - return expectedOutput; -} - -TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, JustDifferentOpsCustomNode) { - std::unique_ptr pipeline; - std::vector input{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - std::vector factors{1, 3, 2, 2}; // add/sub/multiply/divide - this->prepareRequest(request, input, differentOpsInputName); - this->prepareRequest(request, factors, differentOpsFactorsName); - this->loadConfiguration(pipelineCustomNodeDifferentOperationsConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); - prepareDifferentOpsExpectedOutput(expectedOutput, input, factors); - this->checkResponse("pipeline_output", response, expectedOutput, {4, 1, 10}); - - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(pipelineName); - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - - const auto& input_A = inputs.at(differentOpsInputName); - EXPECT_EQ(input_A->getShape(), Shape({1, 10})); - const auto& input_B = inputs.at(differentOpsFactorsName); - EXPECT_EQ(input_B->getShape(), Shape({1, 4})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({4, 1, 10})); -} - -static const char* pipelineCustomNodeDifferentOperationsThenDummyConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_perform_different_operations", - "base_path": "/ovms/bazel-bin/src/lib_node_perform_different_operations.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input", "pipeline_factors"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_perform_different_operations", - "type": "custom", - "demultiply_count": 4, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}}, - {"op_factors": {"node_name": "request", - "data_item": "pipeline_factors"}} - ], - "outputs": [ - {"data_item": "different_ops_results", - "alias": "custom_node_output"} - ] - }, - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "dummyNode", - "data_item": "dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, DifferentOpsCustomNodeThenDummy) { - std::unique_ptr pipeline; - std::vector input{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - std::vector factors{1, 3, 2, 2}; // add/sub/multiply/divide - this->prepareRequest(request, input, differentOpsInputName); - this->prepareRequest(request, factors, differentOpsFactorsName); - this->loadConfiguration(pipelineCustomNodeDifferentOperationsThenDummyConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); - prepareDifferentOpsExpectedOutput(expectedOutput, input, factors); - std::transform(expectedOutput.begin(), expectedOutput.end(), expectedOutput.begin(), - [](float f) -> float { return f + 1; }); - this->checkResponse("pipeline_output", response, expectedOutput, {4, 1, 10}); - - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(pipelineName); - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - - const auto& input_A = inputs.at(pipelineInputName); - EXPECT_EQ(input_A->getShape(), Shape({1, 10})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({4, 1, 10})); -} - -static const char* pipelineCustomNodeDifferentOperations2OutputsConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_perform_different_operations", - "base_path": "/ovms/bazel-bin/src/lib_node_perform_different_operations.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input", "pipeline_factors"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_perform_different_operations", - "type": "custom", - "demultiply_count": 4, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}}, - {"op_factors": {"node_name": "request", - "data_item": "pipeline_factors"}} - ], - "outputs": [ - {"data_item": "different_ops_results", - "alias": "custom_node_output"}, - {"data_item": "different_ops_factors", - "alias": "custom_node_factors"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node", - "data_item": "custom_node_output"} - }, - {"pipeline_factors": {"node_name": "custom_node", - "data_item": "custom_node_factors"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, DifferentOpsCustomNode2OutputsMetadataCheck) { - std::unique_ptr pipeline; - std::vector input{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - std::vector factors{1, 3, 2, 2}; // add/sub/multiply/divide - this->prepareRequest(request, input, differentOpsInputName); - this->prepareRequest(request, factors, differentOpsFactorsName); - this->loadConfiguration(pipelineCustomNodeDifferentOperations2OutputsConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); - prepareDifferentOpsExpectedOutput(expectedOutput, input, factors); - this->checkResponse("pipeline_output", response, expectedOutput, {4, 1, 10}); - - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(pipelineName); - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - - const auto& input_A = inputs.at(differentOpsInputName); - EXPECT_EQ(input_A->getShape(), Shape({1, 10})); - const auto& input_B = inputs.at(differentOpsFactorsName); - EXPECT_EQ(input_B->getShape(), Shape({1, 4})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({4, 1, 10})); - const auto& outputFactors = outputs.at("pipeline_factors"); - EXPECT_EQ(outputFactors->getShape(), Shape({4, 1, 4})); -} - -static const char* pipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_perform_different_operations", - "base_path": "/ovms/bazel-bin/src/lib_node_perform_different_operations.so" - }, - { - "name": "lib_choose_maximum", - "base_path": "/ovms/bazel-bin/src/lib_node_choose_maximum.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input", "pipeline_factors"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_perform_different_operations", - "type": "custom", - "demultiply_count": 4, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}}, - {"op_factors": {"node_name": "request", - "data_item": "pipeline_factors"}} - ], - "outputs": [ - {"data_item": "different_ops_results", - "alias": "custom_node_output"} - ] - }, - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - }, - { - "name": "choose_max", - "library_name": "lib_choose_maximum", - "type": "custom", - "gather_from_node": "custom_node", - "params": { - "selection_criteria": "MAXIMUM_MINIMUM" - }, - "inputs": [ - {"input_tensors": {"node_name": "dummyNode", - "data_item": "dummy_output"}} - ], - "outputs": [ - {"data_item": "maximum_tensor", - "alias": "maximum_tensor_alias"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "choose_max", - "data_item": "maximum_tensor_alias"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, DifferentOpsCustomNodeThenDummyThenChooseMaximum) { - std::unique_ptr pipeline; - std::vector input{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - std::vector factors{1, 3, 2, 2}; // add/sub/multiply/divide - this->prepareRequest(request, input, differentOpsInputName); - this->prepareRequest(request, factors, differentOpsFactorsName); - this->loadConfiguration(pipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); - prepareDifferentOpsExpectedOutput(expectedOutput, input, factors); - std::transform(expectedOutput.begin(), expectedOutput.end(), expectedOutput.begin(), - [](float f) -> float { return f + 1; }); - std::vector expectedResult = prepareGatherHighestExpectedOutput(expectedOutput, Method::MAXIMUM_MINIMUM); - this->checkResponse("pipeline_output", response, expectedResult, {1, 10}); -} - -static const char* pipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumThenDummyConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_perform_different_operations", - "base_path": "/ovms/bazel-bin/src/lib_node_perform_different_operations.so" - }, - { - "name": "lib_choose_maximum", - "base_path": "/ovms/bazel-bin/src/lib_node_choose_maximum.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input", "pipeline_factors"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_perform_different_operations", - "type": "custom", - "demultiply_count": 4, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}}, - {"op_factors": {"node_name": "request", - "data_item": "pipeline_factors"}} - ], - "outputs": [ - {"data_item": "different_ops_results", - "alias": "custom_node_output"} - ] - }, - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - }, - { - "name": "choose_max", - "library_name": "lib_choose_maximum", - "type": "custom", - "gather_from_node": "custom_node", - "params": { - "selection_criteria": "MAXIMUM_MAXIMUM" - }, - "inputs": [ - {"input_tensors": {"node_name": "dummyNode", - "data_item": "dummy_output"}} - ], - "outputs": [ - {"data_item": "maximum_tensor", - "alias": "maximum_tensor_alias"} - ] - }, - { - "name": "dummyNode2", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "choose_max", - "data_item": "maximum_tensor_alias"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "dummyNode2", - "data_item": "dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, DifferentOpsCustomNodeThenDummyThenChooseMaximumThenDummyAgain) { - std::unique_ptr pipeline; - std::vector input{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - std::vector factors{1, 3, 2, 2}; // add/sub/multiply/divide - this->prepareRequest(request, input, differentOpsInputName); - this->prepareRequest(request, factors, differentOpsFactorsName); - this->loadConfiguration(pipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumThenDummyConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); - prepareDifferentOpsExpectedOutput(expectedOutput, input, factors); - std::transform(expectedOutput.begin(), expectedOutput.end(), expectedOutput.begin(), - [](float f) -> float { return f + 1; }); - std::vector expectedResult = prepareGatherHighestExpectedOutput(expectedOutput, Method::MAXIMUM_MAXIMUM); - std::transform(expectedResult.begin(), expectedResult.end(), expectedResult.begin(), - [](float f) -> float { return f + 1; }); - this->checkResponse("pipeline_output", response, expectedResult, {1, 10}); -} - -static const char* demultiplyThenDummyThenChooseMaximumConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_choose_maximum", - "base_path": "/ovms/bazel-bin/src/lib_node_choose_maximum.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "demultiply_count": 0, - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - }, - { - "name": "choose_max", - "library_name": "lib_choose_maximum", - "type": "custom", - "gather_from_node": "request", - "params": { - "selection_criteria": "MAXIMUM_MAXIMUM" - }, - "inputs": [ - {"input_tensors": {"node_name": "dummyNode", - "data_item": "dummy_output"}} - ], - "outputs": [ - {"data_item": "maximum_tensor", - "alias": "maximum_tensor_alias"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "choose_max", - "data_item": "maximum_tensor_alias"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, DemultiplyThenDummyThenChooseMaximum) { - std::unique_ptr pipeline; - std::vector input(4 * DUMMY_MODEL_OUTPUT_SIZE); - std::fill(input.begin(), input.end(), 1.0); - - uint32_t iterations = -1; - uint32_t number = 0; - std::transform(input.begin(), input.end(), input.begin(), - [&iterations, &number](float f) -> float { - iterations++; - number = iterations/10; - return f + number; }); - - this->prepareRequest(request, input, differentOpsInputName, {4, 1, 10}); - this->loadConfiguration(demultiplyThenDummyThenChooseMaximumConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - auto status = pipeline->execute(DEFAULT_TEST_CONTEXT); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - - std::vector expectedOutput{5, 5, 5, 5, 5, 5, 5, 5, 5, 5}; - this->checkResponse("pipeline_output", response, expectedOutput, {1, 10}); -} - -// Extract TensorInfo out of string in format: "1,3,500,500;FP32" -static CustomNodeTensorInfo extractMetadata(const char* key, const char* value) { - std::string keyStr = key; - std::string valueStr = value; - auto tokens = tokenize(valueStr, ';'); - EXPECT_EQ(tokens.size(), 2); - std::string shapeStr = tokens[0]; - std::string precisionStr = tokens[1]; - tokens = tokenize(shapeStr, ','); - EXPECT_GE(tokens.size(), 1); - shape_t shape; - std::transform(tokens.begin(), tokens.end(), std::back_inserter(shape), - [](const std::string& str) { return std::stoull(str); }); - CustomNodeTensorPrecision precision = toCustomNodeTensorPrecision(ovmsPrecisionToIE2Precision(ovms::fromString(precisionStr))); - CustomNodeTensorInfo info; - info.name = key; - info.dimsCount = shape.size(); - info.dims = (uint64_t*)malloc(info.dimsCount * sizeof(uint64_t)); - std::memcpy(info.dims, shape.data(), info.dimsCount * sizeof(uint64_t)); - info.precision = precision; - return info; -} - -struct LibraryParamControlledMetadata { - static bool startsWith(const char* str, const char* prefix) { - // Ensure null terminated - const int MAX = 300; - const char* end = str; - for (; *end != '\0'; ++end) { - if ((end - str) > MAX) { - EXPECT_TRUE(false) << *end; - } - } - const char* end2 = prefix; - for (; *end2 != '\0'; ++end2) { - if ((end2 - prefix) > MAX) { - EXPECT_TRUE(false) << *end2; - } - } - size_t strLen = std::strlen(str); - size_t prefixLen = std::strlen(prefix); - return strLen < prefixLen ? false : std::memcmp(str, prefix, prefixLen) == 0; - } - // Extract TensorInfo out of string in format: "1,3,500,500;FP32" - static CustomNodeTensorInfo extractMetadata(const char* key, const char* value) { - std::string keyStr = key; - std::string valueStr = value; - auto tokens = tokenize(valueStr, ';'); - EXPECT_EQ(tokens.size(), 2); - std::string shapeStr = tokens[0]; - std::string precisionStr = tokens[1]; - tokens = tokenize(shapeStr, ','); - EXPECT_GE(tokens.size(), 0); - shape_t shape; - std::transform(tokens.begin(), tokens.end(), std::back_inserter(shape), - [](const std::string& str) { return std::stoull(str); }); - CustomNodeTensorPrecision precision = toCustomNodeTensorPrecision(ovmsPrecisionToIE2Precision(ovms::fromString(precisionStr))); - CustomNodeTensorInfo info; - info.name = key; - info.dimsCount = shape.size(); - info.dims = (uint64_t*)malloc(info.dimsCount * sizeof(uint64_t)); - std::memcpy(info.dims, shape.data(), info.dimsCount * sizeof(uint64_t)); - info.precision = precision; - return info; - } - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor*, int, struct CustomNodeTensor**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 1; - } - static int getInputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - int inputs = 0; - for (int i = 0; i < paramsCount; i++) { - if (startsWith(params[i].key, "in_")) { - inputs++; - } - } - if (inputs == 0) { - return 1; - } - *infoCount = inputs; - *info = (struct CustomNodeTensorInfo*)malloc(inputs * sizeof(CustomNodeTensorInfo)); - int preparedInputsMetaCount = 0; - for (int i = 0; i < paramsCount; i++) { - if (startsWith(params[i].key, "in_")) { - (*info)[preparedInputsMetaCount] = extractMetadata(params[i].key, params[i].value); - preparedInputsMetaCount++; - } - } - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - int outputs = 0; - for (int i = 0; i < paramsCount; i++) { - if (startsWith(params[i].key, "out_")) { - outputs++; - } - } - if (outputs == 0) { - return 1; - } - *infoCount = outputs; - *info = (struct CustomNodeTensorInfo*)malloc(outputs * sizeof(CustomNodeTensorInfo)); - int preparedInputsMetaCount = 0; - for (int i = 0; i < paramsCount; i++) { - if (startsWith(params[i].key, "out_")) { - (*info)[preparedInputsMetaCount] = extractMetadata(params[i].key, params[i].value); - preparedInputsMetaCount++; - } - } - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -class EnsembleConfigurationValidationWithCustomNode : public ::testing::Test { -protected: - void SetUp() override { - mockedLibrary = createLibraryMock(); - ASSERT_TRUE(mockedLibrary.isValid()); - } - - NodeLibrary mockedLibrary; - - const std::string customNodeInputName = "input_numbers"; - const std::string customNodeOutputName = "output_numbers"; - static constexpr const char* pipelineInputName = "pipeline_input"; - const std::string pipelineOutputName = "pipeline_output"; -}; - -TEST_F(EnsembleConfigurationValidationWithCustomNode, SuccessfulConfiguration) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "1,30,7;I32"}, - {"out_OutputNumbers_2", "1,8;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,30,7;I32"}, - {"in_InputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, SuccessfulConfigurationWithScalar) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", ";FP32"}, - {"out_OutputNumbers_1", ";I32"}, - {"out_OutputNumbers_2", ";I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", ";I32"}, - {"in_InputNumbers_2", ";I32"}, - {"out_OutputNumbers", ";FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, SuccessfulConfigurationWithDynamicShapeInInput) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "1,30,7;I32"}, - {"out_OutputNumbers_2", "1,8;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,0,7;I32"}, - {"in_InputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, SuccessfulConfigurationWithDynamicShapeInOutput) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "1,0,7;I32"}, - {"out_OutputNumbers_2", "1,8;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,30,7;I32"}, - {"in_InputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, ShapesNotMatchBetweenDLModelAndCustomNode) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::DL, "dummy_node_1", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::DL, "dummy_node_2", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10,7;FP32"}, // 1,10 is correcct - {"in_InputNumbers_2", "1,10;FP32"}, - {"out_OutputNumbers", "1,2000;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummy_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["dummy_node_2"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["custom_node"] = { - {"dummy_node_1", {{DUMMY_MODEL_OUTPUT_NAME, "in_InputNumbers_1"}}}, - {"dummy_node_2", {{DUMMY_MODEL_OUTPUT_NAME, "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, ShapesNotMatchBetweenCustomNodeAndDLNode) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10,7;I32"}, - {"out_OutputNumbers", "1,8;FP32"} // 1,10 is correct - }}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers_1"}}}}; - - connections["dummy_node"] = { - {"custom_node", {{"out", DUMMY_MODEL_INPUT_NAME}}}}; - - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, ShapesNotMatchBetweenCustomNodes) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers_1", "1,30,7;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,30,7;I32"}, - {"in_InputNumbers_2", "1,8,1;I32"}, // 1,8 is correct - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, PrecisionNotMatchBetweenDLModelAndCustomNode) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::DL, "dummy_node_1", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::DL, "dummy_node_2", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"in_InputNumbers_2", "1,10;I32"}, // FP32 is correct - {"out_OutputNumbers", "1,2000;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummy_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["dummy_node_2"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["custom_node"] = { - {"dummy_node_1", {{DUMMY_MODEL_OUTPUT_NAME, "in_InputNumbers_1"}}}, - {"dummy_node_2", {{DUMMY_MODEL_OUTPUT_NAME, "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_PRECISION); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, PrecisionNotMatchBetweenCustomNodeAndDLNode) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10,7;I32"}, - {"out_OutputNumbers", "1,10;I32"} // FP32 is correct - }}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers_1"}}}}; - - connections["dummy_node"] = { - {"custom_node", {{"out", DUMMY_MODEL_INPUT_NAME}}}}; - - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_PRECISION); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, PrecisionNotMatchBetweenCustomNodes) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers_1", "1,30,7;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,30,7;FP32"}, // I32 is correct - {"in_InputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_PRECISION); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, NotAllCustomNodeInputsAreConnected) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "1,30,7;I32"}, - {"out_OutputNumbers_2", "1,8;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,30,7;I32"}, - {"in_InputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - // Missing connection {"1", "in_InputNumbers_1"} - connections["custom_node_2"] = { - {"custom_node_1", {{"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_NOT_ALL_INPUTS_CONNECTED); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, CustomNodeMissingOutput) { - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"out", "not_existing_output"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,30,7;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_1", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL_OUTPUT); -} - -TEST_F(EnsembleConfigurationValidationWithCustomNode, InvalidSharedLibrary) { - NodeLibrary invalidLibrary{}; - ASSERT_FALSE(invalidLibrary.isValid()); - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, std::nullopt, {}, invalidLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "1,30,7;I32"}, - {"out_OutputNumbers_2", "1,8;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, invalidLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,30,7;I32"}, - {"in_InputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY); -} - -struct LibraryErrorsOnMetadataCall { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor*, int, struct CustomNodeTensor**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 1; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 1; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleConfigurationValidationWithCustomNode, SharedLibraryErrorsOnMetadataCall) { - NodeLibrary libraryFailingOnMetadataCall = createLibraryMock(); - ASSERT_TRUE(libraryFailingOnMetadataCall.isValid()); - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, std::nullopt, {}, libraryFailingOnMetadataCall, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "1,30,7;I32"}, - {"out_OutputNumbers_2", "1,8;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, libraryFailingOnMetadataCall, - parameters_t{ - {"in_InputNumbers_1", "1,30,7;I32"}, - {"in_InputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::NODE_LIBRARY_METADATA_FAILED); -} - -class EnsembleConfigurationValidationWithDemultiplexer : public EnsembleConfigurationValidationWithCustomNode {}; - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationSingleDemultiplexer) { - const size_t demultiplyCount = 7; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "7,1,700;I32"}, - {"out_OutputNumbers_2", "7,1,8;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,700;I32"}, - {"in_InputNumbers_2", "1,8;FP32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_1"}}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationSingleDynamicDemultiplexerFirst) { - std::optional demultiplyCount = -1; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,10;FP32"}, - {"out_OutputNumbers_1", "0,1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"out_OutputNumbers", "1,10;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_1"}}, - }; - - pipeline_connections_t connections; - connections["custom_node_1"] = {{ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - connections["custom_node_2"] = {{"custom_node_1", {{"1", "in_InputNumbers_1"}}}}; - connections[EXIT_NODE_NAME] = {{"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationSingleDynamicDemultiplexerFixedLibraryFirstMetadataCheck) { - std::optional demultiplyCount = -1; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,10;FP32"}, - {"out_OutputNumbers_1", "12,1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"out_OutputNumbers", "1,10;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_1"}}, - }; - - pipeline_connections_t connections; - connections["custom_node_1"] = {{ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - connections["custom_node_2"] = {{"custom_node_1", {{"1", "in_InputNumbers_1"}}}}; - connections[EXIT_NODE_NAME] = {{"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); - - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - - const auto& input_A = inputs.at(pipelineInputName); - EXPECT_EQ(input_A->getShape(), Shape({1, 10})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({12, 1, 10})); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationFixedDemultiplexerDynamicLibraryFirstMetadataCheckShouldAlsoWarnInLog) { - std::optional demultiplyCount = 12; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,10;FP32"}, - {"out_OutputNumbers_1", "0,1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"out_OutputNumbers", "1,10;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_1"}}, - }; - - pipeline_connections_t connections; - connections["custom_node_1"] = {{ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - connections["custom_node_2"] = {{"custom_node_1", {{"1", "in_InputNumbers_1"}}}}; - connections[EXIT_NODE_NAME] = {{"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); - - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - - const auto& input_A = inputs.at(pipelineInputName); - EXPECT_EQ(input_A->getShape(), Shape({1, 10})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({12, 1, 10})); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationDynamicLibraryShapesMetadataCheckShouldAlsoWarnInLog) { - std::optional demultiplyCount = std::nullopt; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,0,0,0;FP32"}, - {"out_OutputNumbers_1", "0,1,0;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {}}, - }; - - pipeline_connections_t connections; - connections["custom_node_1"] = {{ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - connections[EXIT_NODE_NAME] = {{"custom_node_1", {{"1", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); - - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - - const auto& input_A = inputs.at(pipelineInputName); - EXPECT_EQ(input_A->getShape(), Shape({1, Dimension::any(), Dimension::any(), Dimension::any()})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({Dimension::any(), 1, Dimension::any()})); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationSingleDynamicDemultiplexerLast) { - std::optional demultiplyCount = -1; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,10;FP32"}, - {"out_OutputNumbers_1", "0, 1,10;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_1"}}, - }; - - pipeline_connections_t connections; - connections["custom_node_1"] = {{ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - connections[EXIT_NODE_NAME] = {{"custom_node_1", {{"1", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationSingleDynamicDemultiplexerAndDynamicGather) { - std::optional demultiplyCount = -1; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"out_OutputNumbers_1", "0,1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_2", "1,10;FP32"}, - {"out_OutputNumbers_2", "1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"3", "out_OutputNumbers_3"}}, std::nullopt, {"custom_node_1"}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_3", "0,1,10;FP32"}, - {"out_OutputNumbers_3", "1,10;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - connections["custom_node_1"] = {{ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers_1"}}}}; - connections["custom_node_2"] = {{"custom_node_1", {{"1", "in_InputNumbers_2"}}}}; - connections["custom_node_3"] = {{"custom_node_2", {{"2", "in_InputNumbers_3"}}}}; - connections[EXIT_NODE_NAME] = {{"custom_node_3", {{"3", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationSingleFixedDemultiplexerAndDynamicGather) { - std::optional demultiplyCount = 12; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"out_OutputNumbers_1", "12,1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_2", "1,10;FP32"}, - {"out_OutputNumbers_2", "1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"3", "out_OutputNumbers_3"}}, std::nullopt, {"custom_node_1"}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_3", "0,1,10;FP32"}, - {"out_OutputNumbers_3", "1,10;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - connections["custom_node_1"] = {{ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers_1"}}}}; - connections["custom_node_2"] = {{"custom_node_1", {{"1", "in_InputNumbers_2"}}}}; - connections["custom_node_3"] = {{"custom_node_2", {{"2", "in_InputNumbers_3"}}}}; - connections[EXIT_NODE_NAME] = {{"custom_node_3", {{"3", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationSingleDynamicDemultiplexerAndFixedGatherShouldWarnInLog) { - std::optional demultiplyCount = -1; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"out_OutputNumbers_1", "0,1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"2", "out_OutputNumbers_2"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_2", "1,10;FP32"}, - {"out_OutputNumbers_2", "1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"3", "out_OutputNumbers_3"}}, std::nullopt, {"custom_node_1"}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_3", "12,1,10;FP32"}, - {"out_OutputNumbers_3", "1,10;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - connections["custom_node_1"] = {{ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers_1"}}}}; - connections["custom_node_2"] = {{"custom_node_1", {{"1", "in_InputNumbers_2"}}}}; - connections["custom_node_3"] = {{"custom_node_2", {{"2", "in_InputNumbers_3"}}}}; - connections[EXIT_NODE_NAME] = {{"custom_node_3", {{"3", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationSingleDynamicDemultiplexerFixedLibraryDynamicGatherMetadataCheck) { - std::optional demultiplyCount = -1; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,10;FP32"}, - {"out_OutputNumbers_1", "12,1,10;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"out_OutputNumbers", "1,10;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_1"}}, - }; - - pipeline_connections_t connections; - connections["custom_node_1"] = {{ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - connections["custom_node_2"] = {{"custom_node_1", {{"1", "in_InputNumbers_1"}}}}; - connections[EXIT_NODE_NAME] = {{"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); - - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - - const auto& input_A = inputs.at(pipelineInputName); - EXPECT_EQ(input_A->getShape(), Shape({1, 10})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({12, 1, 10})); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, SuccessfulConfigurationMultipleDemultiplexers) { - const size_t demultiplyCount1 = 11; - const size_t demultiplyCount2 = 43; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount1, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "11,1,700;I32"}, - {"out_OutputNumbers_2", "11,1,8;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount2, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,700;I32"}, - {"in_InputNumbers_2", "1,8;FP32"}, - {"out_OutputNumbers", "43,1,2000;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,2000;FP32"}, - {"out_OutputNumbers", "1,5;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_1", "custom_node_2"}}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections["custom_node_3"] = { - {"custom_node_2", {{"out", "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_3", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, MultipleBatchInCustomNode) { - const size_t demultiplyCount = 9; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "3,3,10;FP32"}, - {"out_OutputNumbers_1", "9,1,700;I32"}, - {"out_OutputNumbers_2", "9,1,8;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,700;I32"}, - {"in_InputNumbers_2", "1,8;FP32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_1"}}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - auto status = pipelineDefinition->validate(manager, manager, manager); - ASSERT_EQ(status, StatusCode::OK) << status.string(); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, DemultiplexerNodeNotEnoughDimensionsToDemultiply) { - const size_t demultiplyCount = 29; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}, demultiplyCount}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"out_OutputNumbers", "25,1,12;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["custom_node"] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, "in_InputNumbers_1"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_NOT_ENOUGH_SHAPE_DIMENSIONS_TO_DEMULTIPLY); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, DemultiplexerCustomNodeNotEnoughDimensionsToDemultiply_Scalar) { - const size_t demultiplyCount = 29; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", ";FP32"}, - {"out_OutputNumbers", ";FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers_1"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_NOT_ENOUGH_SHAPE_DIMENSIONS_TO_DEMULTIPLY); -} - -class DummyModelWithMockedMetadata : public ovms::ModelInstance { - ovms::tensor_map_t mockedInputsInfo, mockedOutputsInfo; - -public: - DummyModelWithMockedMetadata(ov::Core& ieCore, const ovms::tensor_map_t& inputsInfo, const ovms::tensor_map_t& outputsInfo) : - ovms::ModelInstance("dummy", 1, ieCore), - mockedInputsInfo(inputsInfo), - mockedOutputsInfo(outputsInfo) {} - - std::optional getBatchSize() const override { - return 1; - } - - const ovms::ModelConfig& getModelConfig() const override { - return DUMMY_MODEL_CONFIG; - } - - const ovms::tensor_map_t& getInputsInfo() const override { - return mockedInputsInfo; - } - - const ovms::tensor_map_t& getOutputsInfo() const override { - return mockedOutputsInfo; - } -}; - -class ModelWithDummyModelWithMockedMetadata : public ovms::Model { - std::shared_ptr modelInstance; - -public: - ModelWithDummyModelWithMockedMetadata(const std::string& name, std::shared_ptr modelInstance) : - Model(name), - modelInstance(modelInstance) {} - std::shared_ptr modelInstanceFactory(const std::string& modelName, const ovms::model_version_t, ov::Core& ieCore, ovms::MetricRegistry* registry = nullptr, const ovms::MetricConfig* config = nullptr) override { - return modelInstance; - } -}; - -std::shared_ptr dummyModelWithMockedMetadata; - -class ModelManagerWithModelWithDummyModelWithMockedMetadata : public ovms::ModelManager { - std::shared_ptr modelInstance; - -public: - ModelManagerWithModelWithDummyModelWithMockedMetadata(std::shared_ptr modelInstance) : - modelInstance(modelInstance) {} - std::shared_ptr modelFactory(const std::string& name) override { - return std::make_shared("dummy", modelInstance); - } -}; - -struct LibraryCustomNodeWithDemultiplexerAndBatchSizeGreaterThan1ThenDummy { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - if (inputsCount != 1) { - return 1; - } - - if (strcmp(inputs[0].name, "in") != 0) { - return 2; - } - - const struct CustomNodeTensor* input = &inputs[0]; - - *outputsCount = 1; - *outputs = (struct CustomNodeTensor*)malloc(sizeof(struct CustomNodeTensor) * (*outputsCount)); - struct CustomNodeTensor* output = (&(*outputs))[0]; - - output->name = "out"; - output->data = (uint8_t*)malloc(input->dataBytes * sizeof(uint8_t)); - output->dataBytes = input->dataBytes; - memcpy((void*)output->data, (void*)input->data, input->dataBytes * sizeof(uint8_t)); - output->dims = (uint64_t*)malloc(input->dimsCount * sizeof(uint64_t)); - output->dimsCount = input->dimsCount; - memcpy((void*)output->dims, (void*)input->dims, input->dimsCount * sizeof(uint64_t)); - output->precision = input->precision; - return 0; - } - - static int getInputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - std::string name = "input_dims"; - *infoCount = 1; - *info = (struct CustomNodeTensorInfo*)malloc(*infoCount * sizeof(CustomNodeTensorInfo)); - for (int i = 0; i < paramsCount; i++) { - if (params[i].key == name) { - (*info)[0] = extractMetadata(params[i].key, params[i].value); - (*info)->name = "in"; - return 0; - } - } - return 1; - } - static int getOutputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - std::string name = "output_dims"; - *infoCount = 1; - *info = (struct CustomNodeTensorInfo*)malloc(*infoCount * sizeof(CustomNodeTensorInfo)); - for (int i = 0; i < paramsCount; i++) { - if (params[i].key == name) { - (*info)[0] = extractMetadata(params[i].key, params[i].value); - (*info)->name = "out"; - return 0; - } - } - return 1; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, CustomNodeWithDemultiplexerAndBatchSizeGreaterThan1ThenDummy) { - NodeLibrary libraryCustomNodeWithDemultiplexerAndBatchSizeGreaterThan1ThenDummy = createLibraryMock(); - ASSERT_TRUE(libraryCustomNodeWithDemultiplexerAndBatchSizeGreaterThan1ThenDummy.isValid()); - - const size_t demultiplyCount = 7; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{"out", "out"}}, demultiplyCount, {}, libraryCustomNodeWithDemultiplexerAndBatchSizeGreaterThan1ThenDummy, - parameters_t{ - {"input_dims", "7,5,10;FP32"}, - {"output_dims", "7,5,10;FP32"}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node"}}, - }; - - pipeline_connections_t connections; - - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in"}}}}; - - connections["dummy_node"] = { - {"custom_node", {{"out", DUMMY_MODEL_INPUT_NAME}}}}; - - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}}}; - - auto ieCore2 = std::make_unique(); - auto dummyModelInstance = std::make_shared( - *ieCore2, - tensor_map_t{ - {DUMMY_MODEL_INPUT_NAME, std::make_shared( - DUMMY_MODEL_INPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{5, 10})}}, - tensor_map_t{ - {DUMMY_MODEL_OUTPUT_NAME, std::make_shared( - DUMMY_MODEL_OUTPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{5, 10})}}); - - ModelManagerWithModelWithDummyModelWithMockedMetadata manager(dummyModelInstance); - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, CustomNodeWithDemultiplexerAndBatchSizeGreaterThan1ThenDummy) { - // Prepare request - std::vector input(7 * 5 * DUMMY_MODEL_INPUT_SIZE); - std::iota(input.begin(), input.end(), 42); - PredictRequest request; - PredictResponse response; - tensorflow::TensorProto& proto = (*request.mutable_inputs())[pipelineInputName]; - proto.set_dtype(tensorflow::DataType::DT_FLOAT); - proto.mutable_tensor_content()->assign((char*)input.data(), input.size() * sizeof(float)); - proto.mutable_tensor_shape()->add_dim()->set_size(7); - proto.mutable_tensor_shape()->add_dim()->set_size(5); - proto.mutable_tensor_shape()->add_dim()->set_size(10); - - // Prepare model - ConstructorEnabledModelManager manager; - ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchSize(5); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare pipeline - std::optional demultiplyCount = 7; - std::set gather = {"custom_node"}; - std::unordered_map aliases{{"out", "out"}}; - - auto inputTensorInfo = std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{7, 5, 10}, - Layout::getUnspecifiedLayout()); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto tensorInfo = std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{7, 5, 10}, - Layout::getUnspecifiedLayout()); - const tensor_map_t outputsInfo{{pipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo, gather); - auto custom_node = std::make_unique( - "custom_node", - createLibraryMock(), - parameters_t{ - {"input_dims", "7,5,10;FP32"}, - {"output_dims", "7,5,10;FP32"}}, - aliases, demultiplyCount); - auto model_node = std::make_unique("dummy_node", "dummy", std::nullopt, manager); - - auto pipeline = std::make_unique(*input_node, *output_node, *this->reporter); - pipeline->connect(*input_node, *custom_node, {{pipelineInputName, "in"}}); - pipeline->connect(*custom_node, *model_node, {{"out", DUMMY_MODEL_INPUT_NAME}}); - pipeline->connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}); - - pipeline->push(std::move(input_node)); - pipeline->push(std::move(custom_node)); - pipeline->push(std::move(model_node)); - pipeline->push(std::move(output_node)); - - // Execute - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - - // Check response - std::vector expectedOutput = input; - std::transform(expectedOutput.begin(), expectedOutput.end(), expectedOutput.begin(), - [](float f) -> float { return f + 1; }); - this->checkResponse(pipelineOutputName, response, expectedOutput, {7, 5, 10}); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, ShapesNotMatchBetweenDLModelAndCustomNode) { - const size_t demultiplyCount = 33; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}, demultiplyCount}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10;FP32"}, - {"out_OutputNumbers", "1,25,12;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["custom_node"] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, "in_InputNumbers_1"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node", {{"out", pipelineOutputName}}}}; - auto ieCore = std::make_unique(); - auto dummyModelInstance = std::make_shared( - *ieCore, - tensor_map_t{ - {DUMMY_MODEL_INPUT_NAME, std::make_shared( - DUMMY_MODEL_INPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{1, 10})}}, - tensor_map_t{ - {DUMMY_MODEL_OUTPUT_NAME, std::make_shared( - DUMMY_MODEL_OUTPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{demultiplyCount, 1, 11})}}); // demultiplyCount, 1, 10 is correct - - ModelManagerWithModelWithDummyModelWithMockedMetadata manager(dummyModelInstance); - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, ShapesNotMatchBetweenCustomNodeAndDLNode) { - const size_t demultiplyCount = 25; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,10,7;I32"}, - {"out_OutputNumbers", "25,1,12;FP32"} // 25,1,10 is correct - }}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers_1"}}}}; - - connections["dummy_node"] = { - {"custom_node", {{"out", DUMMY_MODEL_INPUT_NAME}}}}; - - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, ShapesNotMatchBetweenCustomNodes) { - const size_t demultiplyCount = 19; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_2", "19,1,8;I32"}, - {"out_OutputNumbers_1", "19,1,30,7;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,30,10;I32"}, // 1,30,7 is correct - {"in_InputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, DemultiplyCountNotMatchingOutputSecondDimensionValue) { - const size_t demultiplyCount = 87; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_2", "87,1,8;I32"}, - {"out_OutputNumbers_1", "86,1,30,7;I32"} // 87,1,30,7 is correct - }}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,30,7;I32"}, - {"in_InputNumbers_2", "1,8;I32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, DemultiplyCountNotMatchingOutputShapeBeforeExitNode) { - const size_t demultiplyCount = 213; - const std::set gatherFrom{"custom_node_1"}; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers", "220,1,30,7;I32"} // 213,1,30,7 is correct - }}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, gatherFrom}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_1", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT); -} - -class EnsembleConfigurationValidationWithGather : public EnsembleConfigurationValidationWithCustomNode {}; - -TEST_F(EnsembleConfigurationValidationWithGather, SuccessfulConfiguration) { - const size_t demultiplyCount = 13; - const std::set gatherFrom{"custom_node_1"}; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "13,1,700;I32"}, - {"out_OutputNumbers_2", "13,1,8;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,700;I32"}, - {"in_InputNumbers_2", "1,8;FP32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, gatherFrom, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "13,1,2000;FP32"}, - {"out_OutputNumbers", "1,5;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections["custom_node_3"] = { - {"custom_node_2", {{"out", "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_3", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithGather, SuccessfulConfigurationWithDLNodeAsDemultliplexer) { - const size_t demultiplyCount = 53; - const std::set gatherFrom{"dummy_node"}; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}, demultiplyCount}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,10;FP32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, gatherFrom, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "53,1,2000;FP32"}, - {"out_OutputNumbers", "1,5;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["custom_node_1"] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"out", "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - auto ieCore = std::make_unique(); - auto dummyModelInstance = std::make_shared( - *ieCore, - tensor_map_t{ - {DUMMY_MODEL_INPUT_NAME, std::make_shared( - DUMMY_MODEL_INPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{1, demultiplyCount, 10})}}, - tensor_map_t{ - {DUMMY_MODEL_OUTPUT_NAME, std::make_shared( - DUMMY_MODEL_OUTPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{demultiplyCount, 1, 10})}}); - - ModelManagerWithModelWithDummyModelWithMockedMetadata manager(dummyModelInstance); - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithGather, SuccessfulConfigurationWithDLNodeAsGather) { - const size_t demultiplyCount = 102; - const std::set gatherFrom{"custom_node_1"}; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,10;FP32"}, - {"out_OutputNumbers", "102,1,2000;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,2000;I32"}, - {"out_OutputNumbers", "1,10;FP32"}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}, std::nullopt, gatherFrom}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"out", "in_InputNumbers"}}}}; - - connections["dummy_node"] = { - {"custom_node_2", {{"out", DUMMY_MODEL_INPUT_NAME}}}}; - - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}}}; - - auto ieCore = std::make_unique(); - auto dummyModelInstance = std::make_shared( - *ieCore, - tensor_map_t{ - {DUMMY_MODEL_INPUT_NAME, std::make_shared( - DUMMY_MODEL_INPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{demultiplyCount, 1, 10})}}, - tensor_map_t{ - {DUMMY_MODEL_OUTPUT_NAME, std::make_shared( - DUMMY_MODEL_OUTPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{1, demultiplyCount, 10})}}); - - ModelManagerWithModelWithDummyModelWithMockedMetadata manager(dummyModelInstance); - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::OK); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, MultipleGathersNotAllowedInNonExitNode) { - const size_t demultiplyCount1 = 11; - const size_t demultiplyCount2 = 43; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount1, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "11,1,700;I32"}, - {"out_OutputNumbers_2", "11,1,8;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount2, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,700;I32"}, - {"in_InputNumbers_2", "1,8;FP32"}, - {"out_OutputNumbers", "43,1,2000;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {"custom_node_1", "custom_node_2"}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "11,43,1,2000;FP32"}, - {"out_OutputNumbers", "1,5;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections["custom_node_3"] = { - {"custom_node_2", {{"out", "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_3", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_MANUAL_GATHERING_FROM_MULTIPLE_NODES_NOT_SUPPORTED); -} - -TEST_F(EnsembleConfigurationValidationWithGather, ShapesNotMatchBetweenDLModelAndCustomNode) { - const size_t demultiplyCount = 53; - const std::set gatherFrom{"dummy_node"}; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}, demultiplyCount}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,10;FP32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, gatherFrom, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "53,1,2000;FP32"}, - {"out_OutputNumbers", "1,5;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["custom_node_1"] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"out", "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{"out", pipelineOutputName}}}}; - - auto ieCore = std::make_unique(); - auto dummyModelInstance = std::make_shared( - *ieCore, - tensor_map_t{ - {DUMMY_MODEL_INPUT_NAME, std::make_shared( - DUMMY_MODEL_INPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{1, demultiplyCount, 10})}}, - tensor_map_t{ - {DUMMY_MODEL_OUTPUT_NAME, std::make_shared( - DUMMY_MODEL_OUTPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{demultiplyCount, 1, 11})}}); // demultiplyCount, 1, 10 is correct - - ModelManagerWithModelWithDummyModelWithMockedMetadata manager(dummyModelInstance); - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleConfigurationValidationWithGather, ShapesNotMatchBetweenCustomNodeAndDLNode) { - const size_t demultiplyCount = 102; - const std::set gatherFrom{"custom_node_1"}; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,10;FP32"}, - {"out_OutputNumbers", "102,1,2000;I32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,2000;I32"}, - {"out_OutputNumbers", "1,10;FP32"}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}, std::nullopt, gatherFrom}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"out", "in_InputNumbers"}}}}; - - connections["dummy_node"] = { - {"custom_node_2", {{"out", DUMMY_MODEL_INPUT_NAME}}}}; - - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}}}; - - auto ieCore = std::make_unique(); - auto dummyModelInstance = std::make_shared( - *ieCore, - tensor_map_t{ - {DUMMY_MODEL_INPUT_NAME, std::make_shared( - DUMMY_MODEL_INPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{demultiplyCount, 1, 11})}}, // 1, demultiplyCount, 10 is correct - tensor_map_t{ - {DUMMY_MODEL_OUTPUT_NAME, std::make_shared( - DUMMY_MODEL_OUTPUT_NAME, - ovms::Precision::FP32, - ovms::Shape{1, demultiplyCount, 10})}}); - - ModelManagerWithModelWithDummyModelWithMockedMetadata manager(dummyModelInstance); - ModelConfig config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleConfigurationValidationWithGather, ShapesNotMatchBetweenCustomNodes) { - const size_t demultiplyCount = 51; - const std::set gatherFrom{"custom_node_1"}; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "51,1,700;I32"}, - {"out_OutputNumbers_2", "51,1,8;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,700;I32"}, - {"in_InputNumbers_2", "1,8;FP32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, gatherFrom, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "51,1,2001;FP32"}, // 51,1,2000 is correct - {"out_OutputNumbers", "1,5;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections["custom_node_3"] = { - {"custom_node_2", {{"out", "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_3", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleConfigurationValidationWithGather, DemultiplyCountNotMatchingInputSecondDimensionValue) { - const size_t demultiplyCount = 94; - const std::set gatherFrom{"custom_node_1"}; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "94,1,700;I32"}, - {"out_OutputNumbers_2", "94,1,8;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,700;I32"}, - {"in_InputNumbers_2", "1,8;FP32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, gatherFrom, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "95,1,2000;FP32"}, // 94,1,2000 is correct - {"out_OutputNumbers", "1,5;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections["custom_node_3"] = { - {"custom_node_2", {{"out", "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_3", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, DemultipliersGatherNodesNotInLIFOOrder) { - const size_t demultiplyCount1 = 11; - const size_t demultiplyCount2 = 43; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"1", "out_OutputNumbers_1"}, {"2", "out_OutputNumbers_2"}}, demultiplyCount1, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers_1", "11,1,700;I32"}, - {"out_OutputNumbers_2", "11,1,8;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount2, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers_1", "1,700;I32"}, - {"in_InputNumbers_2", "1,8;FP32"}, - {"out_OutputNumbers", "43,1,2000;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {"custom_node_1"}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "11,1,2000;FP32"}, - {"out_OutputNumbers", "1,100;I32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_2"}}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"1", "in_InputNumbers_1"}, - {"2", "in_InputNumbers_2"}}}}; - - connections["custom_node_3"] = { - {"custom_node_2", {{"out", "in_InputNumbers"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_3", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, GatherNodeWithoutDemultiplexerPath) { - const size_t demultiplyCount1 = 11; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount1, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers", "11,1,700;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers", "1,700;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers1", "1,700;FP32"}, - {"in_InputNumbers2", "1,700;FP32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {"custom_node_1"}}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_3"] = { - {"custom_node_1", {{"out", "in_InputNumbers1"}}}, - {"custom_node_2", {{"out", "in_InputNumbers2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_3", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER); -} - -TEST_F(EnsembleConfigurationValidationWithDemultiplexer, DemultiplexerWithoutGatherNodePath) { - const size_t demultiplyCount1 = 11; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node_1", "", std::nullopt, {{"out", "out_OutputNumbers"}}, demultiplyCount1, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,3,10;FP32"}, - {"out_OutputNumbers", "11,1,700;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {"custom_node_1"}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "11,1,700;FP32"}, - {"out_OutputNumbers", "1,700;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers", "1,700;FP32"}, - {"out_OutputNumbers", "1,700;FP32"}}}, - {NodeKind::CUSTOM, "custom_node_4", "", std::nullopt, {{"out", "out_OutputNumbers"}}, std::nullopt, {}, mockedLibrary, - parameters_t{ - {"in_InputNumbers1", "1,700;FP32"}, - {"in_InputNumbers2", "1,700;FP32"}, - {"out_OutputNumbers", "1,2000;FP32"}}}, - {NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, {}}, - }; - - pipeline_connections_t connections; - - connections["custom_node_1"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, "in_InputNumbers"}}}}; - - connections["custom_node_2"] = { - {"custom_node_1", {{"out", "in_InputNumbers"}}}}; - - connections["custom_node_3"] = { - {"custom_node_1", {{"out", "in_InputNumbers"}}}}; - - connections["custom_node_4"] = { - {"custom_node_2", {{"out", "in_InputNumbers1"}}}, - {"custom_node_3", {{"out", "in_InputNumbers2"}}}}; - - connections[EXIT_NODE_NAME] = { - {"custom_node_4", {{"out", pipelineOutputName}}}}; - - ConstructorEnabledModelManager manager; - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validate(manager, manager, manager), StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER); -} - -class EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest : public EnsembleFlowCustomNodeLoadConfigThenExecuteTest { -protected: - void SetUp() override { - EnsembleFlowCustomNodeLoadConfigThenExecuteTest::SetUp(); - configJsonFilePath = directoryPath + "/ovms_config_file.json"; - } - const std::string differentOpsInputName = "pipeline_input"; -}; - -static const char* pipelineCustomNodeDynamicDemultiplexThenDummyConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_dynamic_demultiplex", - "base_path": "/ovms/bazel-bin/src/lib_node_dynamic_demultiplex.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input", "pipeline_factors"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_dynamic_demultiplex", - "type": "custom", - "demultiply_count": 0, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "dynamic_demultiplex_results", - "alias": "custom_node_output"} - ] - }, - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "dummyNode", - "data_item": "dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, JustDynamicDemultiplexerConfig) { - std::unique_ptr pipeline; - uint8_t dynamicDemultiplyCount = 3; - std::vector input{static_cast(dynamicDemultiplyCount), 1, 2, 3, 4, 5, 6, 7, 8, 9}; - this->prepareRequest(request, input, differentOpsInputName); - this->loadConfiguration(pipelineCustomNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - std::vector expectedOutput(dynamicDemultiplyCount * DUMMY_MODEL_OUTPUT_SIZE); - for (size_t i = 0; i < dynamicDemultiplyCount; ++i) { - std::copy(input.begin(), input.end(), expectedOutput.begin() + i * DUMMY_MODEL_OUTPUT_SIZE); - } - std::transform(expectedOutput.begin(), expectedOutput.end(), expectedOutput.begin(), - [](float f) -> float { return f + 1; }); - this->checkResponse("pipeline_output", response, expectedOutput, {dynamicDemultiplyCount, 1, 10}); - - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(pipelineName); - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - auto& input_A = inputs.at(pipelineInputName); - EXPECT_EQ(input_A->getShape(), Shape({1, 10})); - auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({Dimension::any(), 1, 10})); - - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuardPtr; - auto status = manager.getModelInstance("dummy", 1, modelInstance, modelInstanceUnloadGuardPtr); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - tensor_map_t modelInputs = modelInstance->getInputsInfo(); - tensor_map_t modelOutputs = modelInstance->getOutputsInfo(); - ASSERT_NE(modelInputs.find("b"), modelInputs.end()); - ASSERT_NE(modelOutputs.find("a"), modelOutputs.end()); - auto inputDummy = modelInputs.at("b"); - EXPECT_EQ(inputDummy->getShape(), Shape({1, 10})); - auto outputDummy = modelOutputs.at("a"); - EXPECT_EQ(outputDummy->getShape(), Shape({1, 10})); - - modelInputs.clear(); - modelOutputs.clear(); - - auto inputs2 = pipelineDefinition->getInputsInfo(); - auto outputs2 = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs2.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs2.find(pipelineOutputName), outputs.end()); - auto input_A2 = inputs2.at(pipelineInputName); - EXPECT_EQ(input_A2->getShape(), Shape({1, 10})); - auto output2 = outputs2.at(pipelineOutputName); - EXPECT_EQ(output2->getShape(), Shape({Dimension::any(), 1, 10})); - - status = manager.getModelInstance("dummy", 1, modelInstance, modelInstanceUnloadGuardPtr); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - modelInputs = modelInstance->getInputsInfo(); - modelOutputs = modelInstance->getOutputsInfo(); - ASSERT_NE(modelInputs.find("b"), modelInputs.end()); - ASSERT_NE(modelOutputs.find("a"), modelOutputs.end()); - auto inputDummy2 = modelInputs.at("b"); - EXPECT_EQ(inputDummy2->getShape(), Shape({1, 10})); - auto outputDummy2 = modelOutputs.at("a"); - EXPECT_EQ(outputDummy2->getShape(), Shape({1, 10})); -} - -static const char* pipelineCustomNodeDynamicDemultiplexThenDummyDemultiplexerConnectedToExitConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_dynamic_demultiplex", - "base_path": "/ovms/bazel-bin/src/lib_node_dynamic_demultiplex.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input", "pipeline_factors"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_dynamic_demultiplex", - "type": "custom", - "demultiply_count": 0, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "dynamic_demultiplex_results", - "alias": "custom_node_output"} - ] - }, - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "dummyNode", - "data_item": "dummy_output"} - }, - {"pipeline_output2": {"node_name": "custom_node", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, JustDynamicDemultiplexerThenDummyBothConnectedToExitConfigMetadataCheck) { - this->loadConfiguration(pipelineCustomNodeDynamicDemultiplexThenDummyDemultiplexerConnectedToExitConfig); - - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(pipelineName); - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - const auto& input_A = inputs.at(pipelineInputName); - EXPECT_EQ(input_A->getShape(), Shape({1, 10})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({Dimension::any(), 1, 10})); - const auto& output2 = outputs.at(pipelineOutputName + "2"); - EXPECT_EQ(output2->getShape(), Shape({Dimension::any(), 1, 10})); -} - -static const char* pipelineEntryNodeDynamicDemultiplexThenDummyConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_dynamic_demultiplex", - "base_path": "/ovms/bazel-bin/src/lib_node_dynamic_demultiplex.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "demultiply_count": 0, - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "dummyNode", - "data_item": "dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, DynamicDemultiplexerEntryThenDummyConfig) { - std::unique_ptr pipeline; - uint8_t dynamicDemultiplyCount = 3; - std::vector input(3 * DUMMY_MODEL_OUTPUT_SIZE); - std::iota(input.begin(), input.end(), 42); - this->prepareRequest(request, input, differentOpsInputName, {dynamicDemultiplyCount, 1, 10}); - this->loadConfiguration(pipelineEntryNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - std::vector expectedOutput = input; - std::transform(expectedOutput.begin(), expectedOutput.end(), expectedOutput.begin(), - [](float f) -> float { return f + 1; }); - this->checkResponse("pipeline_output", response, expectedOutput, {dynamicDemultiplyCount, 1, 10}); -} - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, DynamicDemultiplexerEntryMetadataCorrectness) { - this->loadConfiguration(pipelineEntryNodeDynamicDemultiplexThenDummyConfig); - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(pipelineName); - ASSERT_NE(pipelineDefinition, nullptr); - - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - - const auto& input_A = inputs.at(pipelineInputName); - EXPECT_EQ(input_A->getShape(), Shape({Dimension::any(), 1, 10})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({Dimension::any(), 1, 10})); -} - -static const char* pipelineEntryNodeDemultiplexThenDummyConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(5, 10) ", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "demultiply_count": 3, - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "dummyNode", - "data_item": "dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, DemultiplexerEntryThenDummyConfig) { - std::unique_ptr pipeline; - std::vector input(3 * 5 * DUMMY_MODEL_INPUT_SIZE); - std::iota(input.begin(), input.end(), 42); - this->prepareRequest(request, input, pipelineInputName, {3, 5, DUMMY_MODEL_INPUT_SIZE}); - this->loadConfiguration(pipelineEntryNodeDemultiplexThenDummyConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - std::vector expectedOutput = input; - std::transform(expectedOutput.begin(), expectedOutput.end(), expectedOutput.begin(), - [](float f) -> float { return f + 1; }); - this->checkResponse(pipelineOutputName, response, expectedOutput, {3, 5, DUMMY_MODEL_OUTPUT_SIZE}); -} - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, DemultiplexerEntryThenDummMetadataCorrectness) { - this->loadConfiguration(pipelineEntryNodeDemultiplexThenDummyConfig); - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(pipelineName); - ASSERT_NE(pipelineDefinition, nullptr); - - auto inputs = pipelineDefinition->getInputsInfo(); - auto outputs = pipelineDefinition->getOutputsInfo(); - ASSERT_NE(inputs.find(pipelineInputName), inputs.end()); - ASSERT_NE(outputs.find(pipelineOutputName), outputs.end()); - - const auto& input = inputs.at(pipelineInputName); - EXPECT_EQ(input->getShape(), Shape({3, 5, DUMMY_MODEL_INPUT_SIZE})); - const auto& output = outputs.at(pipelineOutputName); - EXPECT_EQ(output->getShape(), Shape({3, 5, DUMMY_MODEL_OUTPUT_SIZE})); -} - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, DynamicDemultiplexerHittingLimitShouldReturnError) { - std::unique_ptr pipeline; - const uint64_t demultiplyLimit = 10'000; // node.cpp - uint64_t dynamicDemultiplyCount = demultiplyLimit + 1; - ASSERT_GT(dynamicDemultiplyCount, demultiplyLimit) << "Current demultiply count type"; - std::vector input{static_cast(dynamicDemultiplyCount), 1, 2, 3, 4, 5, 6, 7, 8, 9}; - this->prepareRequest(request, input, differentOpsInputName); - this->loadConfiguration(pipelineCustomNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - auto status = pipeline->execute(DEFAULT_TEST_CONTEXT); - ASSERT_EQ(status, StatusCode::PIPELINE_TOO_LARGE_DIMENSION_SIZE_TO_DEMULTIPLY) << status.string(); -} - -static const char* pipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumNotInOrderConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_perform_different_operations", - "base_path": "/ovms/bazel-bin/src/lib_node_perform_different_operations.so" - }, - { - "name": "lib_choose_maximum", - "base_path": "/ovms/bazel-bin/src/lib_node_choose_maximum.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input", "pipeline_factors"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - }, - { - "name": "choose_max", - "library_name": "lib_choose_maximum", - "type": "custom", - "gather_from_node": "custom_node", - "params": { - "selection_criteria": "MAXIMUM_MINIMUM" - }, - "inputs": [ - {"input_tensors": {"node_name": "dummyNode", - "data_item": "dummy_output"}} - ], - "outputs": [ - {"data_item": "maximum_tensor", - "alias": "maximum_tensor_alias"} - ] - }, - { - "name": "custom_node", - "library_name": "lib_perform_different_operations", - "type": "custom", - "demultiply_count": 4, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}}, - {"op_factors": {"node_name": "request", - "data_item": "pipeline_factors"}} - ], - "outputs": [ - {"data_item": "different_ops_results", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "choose_max", - "data_item": "maximum_tensor_alias"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, DifferentOpsCustomNodeThenDummyThenChooseMaximumNotInOrderConfig) { - std::unique_ptr pipeline; - std::vector input{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - std::vector factors{1, 3, 2, 2}; // add/sub/multiply/divide - this->prepareRequest(request, input, differentOpsInputName); - this->prepareRequest(request, factors, differentOpsFactorsName); - this->loadConfiguration(pipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumNotInOrderConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); - prepareDifferentOpsExpectedOutput(expectedOutput, input, factors); - std::transform(expectedOutput.begin(), expectedOutput.end(), expectedOutput.begin(), - [](float f) -> float { return f + 1; }); - std::vector expectedResult = prepareGatherHighestExpectedOutput(expectedOutput, Method::MAXIMUM_MINIMUM); - this->checkResponse("pipeline_output", response, expectedResult, {1, 10}); -} - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, DynamicDemultiplexerNoResults) { - std::unique_ptr pipeline; - uint8_t dynamicDemultiplyCount = 0; - std::vector input{static_cast(dynamicDemultiplyCount), 1, 2, 3, 4, 5, 6, 7, 8, 9}; - this->prepareRequest(request, input, differentOpsInputName); - this->loadConfiguration(pipelineCustomNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS); -} - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, DISABLED_JustDynamicDemultiplexerConfigReturning0Batch) { - std::unique_ptr pipeline; - uint8_t dynamicDemultiplyCount = 0; - std::vector input{static_cast(dynamicDemultiplyCount), 1, 2, 3, 4, 5, 6, 7, 8, 9}; - this->prepareRequest(request, input, differentOpsInputName); - this->loadConfiguration(pipelineCustomNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - std::vector expectedOutput(dynamicDemultiplyCount * DUMMY_MODEL_OUTPUT_SIZE); - for (size_t i = 0; i < dynamicDemultiplyCount; ++i) { - std::copy(input.begin(), input.end(), expectedOutput.begin() + i * DUMMY_MODEL_OUTPUT_SIZE); - } - std::transform(expectedOutput.begin(), expectedOutput.end(), expectedOutput.begin(), - [](float f) -> float { return f + 1; }); - this->checkResponse("pipeline_output", response, expectedOutput, {1, dynamicDemultiplyCount, 10}); -} - -static const char* pipelineCustomNode2DynamicDemultiplexConfig = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_dynamic_demultiplex", - "base_path": "/ovms/bazel-bin/src/lib_node_dynamic_demultiplex.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input", "pipeline_factors"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_dynamic_demultiplex", - "type": "custom", - "demultiply_count": 0, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "dynamic_demultiplex_results", - "alias": "custom_node_output"} - ] - }, - { - "name": "custom_node2", - "library_name": "lib_dynamic_demultiplex", - "type": "custom", - "demultiply_count": 0, - "inputs": [ - {"input_numbers": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "dynamic_demultiplex_results", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node2", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, 2DynamicDemultiplexersNotAllowed) { - std::unique_ptr pipeline; - this->loadConfiguration(pipelineCustomNode2DynamicDemultiplexConfig, StatusCode::NOT_IMPLEMENTED); -} - -struct LibraryProduceImages5Dimensions { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor* inputs, int, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - const CustomNodeTensor& input = *inputs; - std::vector inputData((float*)input.data, ((float*)input.data) + (input.dataBytes / sizeof(float))); - - *outputsCount = 1; - int elements = 3 * 1 * 1 * 2 * 3; - *outputs = (struct CustomNodeTensor*)malloc(*outputsCount * sizeof(CustomNodeTensor)); - float* result = (float*)malloc(elements * sizeof(float)); - std::vector data; - for (size_t i = 0; i < 3; i++) { - for (float v : inputData) { - data.push_back(v + float(i) + 1.0); - } - } - std::memcpy(result, data.data(), elements * sizeof(float)); - - CustomNodeTensor& resultTensor = (*outputs)[0]; - resultTensor.name = "custom_node_output"; - resultTensor.data = reinterpret_cast(result); - resultTensor.dimsCount = 5; - resultTensor.dims = (uint64_t*)malloc(resultTensor.dimsCount * sizeof(uint64_t)); - resultTensor.dims[0] = 3; - resultTensor.dims[1] = 1; - resultTensor.dims[2] = 1; - resultTensor.dims[3] = 2; - resultTensor.dims[4] = 3; - resultTensor.dataBytes = elements * sizeof(float); - resultTensor.precision = FP32; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, DemultiplexerConnectedToNhwcNodeDynamicDemultiply_NegativeOne) { - // Prepare request - const std::vector inputValues{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - PredictRequest request; - PredictResponse response; - tensorflow::TensorProto& proto = (*request.mutable_inputs())[pipelineInputName]; - proto.set_dtype(tensorflow::DataType::DT_FLOAT); - proto.mutable_tensor_content()->assign((char*)inputValues.data(), inputValues.size() * sizeof(float)); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(3); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(2); - - // Prepare model - ConstructorEnabledModelManager manager; - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,1,2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare pipeline - std::optional demultiplyCount = -1; - std::set gather = {"image_demultiplexer_node"}; - std::unordered_map aliases{{"custom_node_output", "custom_node_output"}}; - - auto inputTensorInfo = std::make_shared(pipelineOutputName, - Precision::FP32, - Shape{Dimension::any(), 3, 1, 2}); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto tensorInfo = std::make_shared(pipelineOutputName, - Precision::FP32, - Shape{Dimension::any(), 1, 3, 1, 2}); - const tensor_map_t outputsInfo{{pipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo, gather); - auto custom_node = std::make_unique( - "image_demultiplexer_node", - createLibraryMock(), - parameters_t{}, aliases, demultiplyCount); - auto model_node = std::make_unique("increment_node", "increment_1x3x4x5", std::nullopt, manager); - - auto pipeline = std::make_unique(*input_node, *output_node, *this->reporter); - pipeline->connect(*input_node, *custom_node, {{pipelineInputName, "any"}}); - pipeline->connect(*custom_node, *model_node, {{"custom_node_output", "input"}}); - pipeline->connect(*model_node, *output_node, {{"output", pipelineOutputName}}); - - pipeline->push(std::move(input_node)); - pipeline->push(std::move(custom_node)); - pipeline->push(std::move(model_node)); - pipeline->push(std::move(output_node)); - - // Execute - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - checkIncrement4DimResponse(pipelineOutputName, {3.0, 6.0, 4.0, 7.0, 5.0, 8.0, 4.0, 7.0, 5.0, 8.0, 6.0, 9.0, 5.0, 8.0, 6.0, 9.0, 7.0, 10.0}, response, {3, 1, 3, 1, 2}); -} - -struct LibraryProduceImages5DimensionsInFP32OutFP64 { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor* inputs, int, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - const CustomNodeTensor& input = *inputs; - std::vector inputData((float*)input.data, ((float*)input.data) + (input.dataBytes / sizeof(float))); - - *outputsCount = 1; - int elements = 3 * 1 * 1 * 2 * 3; - *outputs = (struct CustomNodeTensor*)malloc(*outputsCount * sizeof(CustomNodeTensor)); - double* result = (double*)malloc(elements * sizeof(double)); - std::vector data; - for (size_t i = 0; i < 3; i++) { - for (float v : inputData) { - data.push_back(double(v) + double(i) + 1.0); - } - } - std::memcpy(result, data.data(), elements * sizeof(double)); - - CustomNodeTensor& resultTensor = (*outputs)[0]; - resultTensor.name = "custom_node_output"; - resultTensor.data = reinterpret_cast(result); - resultTensor.dimsCount = 5; - resultTensor.dims = (uint64_t*)malloc(resultTensor.dimsCount * sizeof(uint64_t)); - resultTensor.dims[0] = 3; - resultTensor.dims[1] = 1; - resultTensor.dims[2] = 1; - resultTensor.dims[3] = 2; - resultTensor.dims[4] = 3; - resultTensor.dataBytes = elements * sizeof(double); - resultTensor.precision = FP64; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, DemultiplexerConnectedToNhwcNode) { - // Prepare request - const std::vector inputValues{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - PredictRequest request; - PredictResponse response; - tensorflow::TensorProto& proto = (*request.mutable_inputs())[pipelineInputName]; - proto.set_dtype(tensorflow::DataType::DT_FLOAT); - proto.mutable_tensor_content()->assign((char*)inputValues.data(), inputValues.size() * sizeof(float)); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(3); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(2); - - // Prepare model - ConstructorEnabledModelManager manager; - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,1,2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(config.parseLayoutParameter("nhwc:nchw"), ovms::StatusCode::OK); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare pipeline - std::optional demultiplyCount = -1; - std::set gather = {"image_demultiplexer_node"}; - std::unordered_map aliases{{"custom_node_output", "custom_node_output"}}; - - auto inputTensorInfo = std::make_shared(pipelineOutputName, - Precision::FP32, - Shape{Dimension::any(), 3, 1, 2}); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto tensorInfo = std::make_shared(pipelineOutputName, - Precision::FP32, - Shape{Dimension::any(), 1, 3, 1, 2}); - const tensor_map_t outputsInfo{{pipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo, gather); - auto custom_node = std::make_unique( - "image_demultiplexer_node", - createLibraryMock(), - parameters_t{}, aliases, demultiplyCount); - auto model_node = std::make_unique("increment_node", "increment_1x3x4x5", std::nullopt, manager); - - auto pipeline = std::make_unique(*input_node, *output_node, *this->reporter); - pipeline->connect(*input_node, *custom_node, {{pipelineInputName, "any"}}); - pipeline->connect(*custom_node, *model_node, {{"custom_node_output", "input"}}); - pipeline->connect(*model_node, *output_node, {{"output", pipelineOutputName}}); - - pipeline->push(std::move(input_node)); - pipeline->push(std::move(custom_node)); - pipeline->push(std::move(model_node)); - pipeline->push(std::move(output_node)); - - // Execute - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - checkIncrement4DimResponse(pipelineOutputName, {3.0, 6.0, 4.0, 7.0, 5.0, 8.0, 4.0, 7.0, 5.0, 8.0, 6.0, 9.0, 5.0, 8.0, 6.0, 9.0, 7.0, 10.0}, response, {3, 1, 3, 1, 2}); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, DemultiplexerCreatesShardedFP64TensorsFromCustomNode) { - /* - Description: - - Entry (1x3x1x2, fp32) ----------> (1x3x1x2, fp32) CustomNode (3x1x3x1x2, fp64) --- demultiplexer -------> (1x3x1x2, fp64) 3x ModelNode (1x3x1x2, fp64) ----- gather -----> (3x1x3x1x2, fp64) Exit - */ - - // Prepare request - const std::vector inputValues{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - PredictRequest request; - PredictResponse response; - tensorflow::TensorProto& proto = (*request.mutable_inputs())[pipelineInputName]; - proto.set_dtype(tensorflow::DataType::DT_FLOAT); - proto.mutable_tensor_content()->assign((char*)inputValues.data(), inputValues.size() * sizeof(float)); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(3); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(2); - - // Prepare model - ConstructorEnabledModelManager manager; - ModelConfig config = DUMMY_FP64_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,1,2,3)"), ovms::StatusCode::OK); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare pipeline - std::optional demultiplyCount = -1; - std::set gather = {"image_demultiplexer_node"}; - std::unordered_map aliases{{"custom_node_output", "custom_node_output"}}; - - auto inputTensorInfo = std::make_shared(pipelineOutputName, - Precision::FP32, - Shape{Dimension::any(), 3, 1, 2}); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto tensorInfo = std::make_shared(pipelineOutputName, - Precision::FP64, - Shape{Dimension::any(), 1, 1, 2, 3}); - const tensor_map_t outputsInfo{{pipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo, gather); - auto custom_node = std::make_unique( - "image_demultiplexer_node", - createLibraryMock(), - parameters_t{}, aliases, demultiplyCount); - auto model_node = std::make_unique("increment_node", "dummy_fp64", std::nullopt, manager); - - auto pipeline = std::make_unique(*input_node, *output_node, *this->reporter); - pipeline->connect(*input_node, *custom_node, {{pipelineInputName, "any"}}); - pipeline->connect(*custom_node, *model_node, {{"custom_node_output", "input:0"}}); - pipeline->connect(*model_node, *output_node, {{"output:0", pipelineOutputName}}); - - pipeline->push(std::move(input_node)); - pipeline->push(std::move(custom_node)); - pipeline->push(std::move(model_node)); - pipeline->push(std::move(output_node)); - - // Execute - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - checkIncrement4DimResponse(pipelineOutputName, {3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}, response, {3, 1, 1, 2, 3}); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, FirstZeroDimWithDemultiplexer) { - // input (D)DL(-1,1,1,1) DL(1,1,1) output - // ====[0,1,1,1]===>O---------->O-----0x[1,1,1]----STOP - ConstructorEnabledModelManager manager; - - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(-1,1,1,1)"), ovms::StatusCode::OK); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,1,1)"), ovms::StatusCode::OK); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - const std::vector inputValues; - this->prepareRequest(this->request, inputValues, pipelineInputName, {0, 1, 1, 1}); - - const tensor_map_t inputsInfo{{pipelineInputName, std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), 1, 1, 1}, - Layout{"N..."})}}; - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), 1, 1, 1}, - Layout{"N..."})}}; - auto output_node = std::make_unique>(&response, outputsInfo, std::set{"node_1"}); - std::optional demultiplyCount{-1}; - auto model_1 = std::make_unique( - "node_1", - "increment_1x3x4x5", - std::nullopt, - manager, std::unordered_map{}, demultiplyCount); - auto model_2 = std::make_unique( - "node_2", - "dummy", - std::nullopt, - manager); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_1, {{pipelineInputName, INCREMENT_1x3x4x5_MODEL_INPUT_NAME}}); - pipeline.connect(*model_1, *model_2, {{INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_2, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_1)); - pipeline.push(std::move(model_2)); - pipeline.push(std::move(output_node)); - - // Expect 0 first dimension to cause pipeline to stop - // In future we could gather 0 elements and prepare [0,1,1,1] out of that - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, GatheringZeroDimension) { - // input (D)DL(-1,1,1,-1) DL(1,1,1) output - // ====[2,1,1,0]===>O---------->O-----2x[1,1,0]---------------->O----[2,1,1,0]--->0 - ConstructorEnabledModelManager manager; - - ModelConfig config = INCREMENT_1x3x4x5_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(-1,1,1,-1)"), ovms::StatusCode::OK); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,1,-1)"), ovms::StatusCode::OK); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - const std::vector inputValues; - this->prepareRequest(this->request, inputValues, pipelineInputName, {2, 1, 1, 0}); - - const tensor_map_t inputsInfo{{pipelineInputName, std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), 1, 1, ovms::Dimension::any()}, - Layout{"N..."})}}; - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), 1, 1, ovms::Dimension::any()}, - Layout{"N..."})}}; - auto output_node = std::make_unique>(&response, outputsInfo, std::set{"node_1"}); - std::optional demultiplyCount{-1}; - auto model_1 = std::make_unique( - "node_1", - "increment_1x3x4x5", - std::nullopt, - manager, std::unordered_map{}, demultiplyCount); - auto model_2 = std::make_unique( - "node_2", - "dummy", - std::nullopt, - manager); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_1, {{pipelineInputName, INCREMENT_1x3x4x5_MODEL_INPUT_NAME}}); - pipeline.connect(*model_1, *model_2, {{INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_2, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_1)); - pipeline.push(std::move(model_2)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse(pipelineOutputName, {}, response, {2, 1, 1, 0}); -} - -// Accepting static input [0,10], producing [1,10] (0.0f, 1.0f, ...) -struct LibraryWithZeroDimInput { - static constexpr float libraryScalarNodeAddValue = 2.1f; - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** handle, int* outputsNum, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - if (inputsCount != 1) - return 1; - if (inputs[0].dimsCount != 2) - return 2; - if (inputs[0].dims[0] != 0 || inputs[0].dims[1] != 10) - return 3; - *handle = (struct CustomNodeTensor*)malloc(sizeof(struct CustomNodeTensor)); - *outputsNum = 1; - (*handle)->name = "output_numbers"; - (*handle)->precision = CustomNodeTensorPrecision::FP32; - (*handle)->dimsCount = 2; - (*handle)->dims = (uint64_t*)malloc(sizeof(uint64_t) * (*handle)->dimsCount); - (*handle)->dims[0] = 1; - (*handle)->dims[1] = 10; - (*handle)->dataBytes = 10 * sizeof(float); - (*handle)->data = (uint8_t*)malloc((*handle)->dataBytes); - for (int i = 0; i < 10; i++) - *(((float*)(*handle)->data) + i) = (float)i; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, AcceptDimZeroInCustomNode) { - // input DL(-1,-1) CN DL(-1,-1) output - // ====[0,10]==>O------->O---[0,10]--->O---[1,10]----->O---[1,10]---->O - ConstructorEnabledModelManager modelManager; - ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(-1,-1)"); - modelManager.reloadModelWithVersions(config); - - const std::vector inputValues; - const std::vector expectedOutputValues; - this->prepareRequest(this->request, inputValues, pipelineInputName, {0, 10}); - - const tensor_map_t inputsInfo{{pipelineInputName, std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), ovms::Dimension::any()}, - Layout{"N..."})}}; - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), ovms::Dimension::any()}, - Layout{"N..."})}}; - auto output_node = std::make_unique>(&response, outputsInfo); - auto model_node_before = std::make_unique( - "dummy_node_before", - "dummy", - std::nullopt, - modelManager); - auto custom_node = std::make_unique("custom_node_0", createLibraryMock(), - parameters_t{}); - auto model_node_after = std::make_unique( - "dummy_node_after", - "dummy", - std::nullopt, - modelManager); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node_before, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_before, *custom_node, {{DUMMY_MODEL_OUTPUT_NAME, "anything"}}); - pipeline.connect(*custom_node, *model_node_after, {{"output_numbers", DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_after, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(custom_node)); - pipeline.push(std::move(model_node_before)); - pipeline.push(std::move(model_node_after)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkResponse(pipelineOutputName, this->response, std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f}, shape_t{1, 10}); -} - -// Accepting any input, producing [5,0] -struct LibraryWithZeroDimOutput { - static constexpr float libraryScalarNodeAddValue = 2.1f; - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** handle, int* outputsNum, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - if (inputsCount != 1) - return 1; - *handle = (struct CustomNodeTensor*)malloc(sizeof(struct CustomNodeTensor)); - *outputsNum = 1; - (*handle)->name = "output_numbers"; - (*handle)->precision = CustomNodeTensorPrecision::FP32; - (*handle)->dimsCount = 2; - (*handle)->dims = (uint64_t*)malloc(sizeof(uint64_t) * (*handle)->dimsCount); - (*handle)->dims[0] = 5; - (*handle)->dims[1] = 0; - (*handle)->data = (uint8_t*)malloc(0); - (*handle)->dataBytes = 0; - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, ReturnDimZeroFromCustomNode) { - // input DL(-1,-1) CN DL(-1,-1) output - // ====[1,1]===>O------->O---[1,1]---->O---[0,5]----->O---[0,5]---->O - // |___[0,5]___________________> - ConstructorEnabledModelManager modelManager; - ModelConfig config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(-1,-1)"); - modelManager.reloadModelWithVersions(config); - const std::string outputNameFromCustomNode{"second_output"}; - - const std::vector inputValues{5.4f}; - const std::vector expectedOutputValues; - this->prepareRequest(this->request, inputValues, pipelineInputName, {1, 1}); - - const tensor_map_t inputsInfo{{pipelineInputName, std::make_shared(pipelineInputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), ovms::Dimension::any()}, - Layout{"N..."})}}; - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{pipelineOutputName, std::make_shared(pipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), ovms::Dimension::any()}, - Layout{"N..."})}, - {outputNameFromCustomNode, std::make_shared(outputNameFromCustomNode, - ovms::Precision::FP32, - ovms::Shape{5, 0}, - Layout{"..."})}}; - auto output_node = std::make_unique>(&response, outputsInfo); - auto model_node_before = std::make_unique( - "dummy_node_before", - "dummy", - std::nullopt, - modelManager); - auto custom_node = std::make_unique("custom_node_0", createLibraryMock(), - parameters_t{}); - auto model_node_after = std::make_unique( - "dummy_node_after", - "dummy", - std::nullopt, - modelManager); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node_before, {{pipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_before, *custom_node, {{DUMMY_MODEL_OUTPUT_NAME, "anything"}}); - pipeline.connect(*custom_node, *model_node_after, {{"output_numbers", DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_after, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, pipelineOutputName}}); - pipeline.connect(*custom_node, *output_node, {{"output_numbers", outputNameFromCustomNode}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(custom_node)); - pipeline.push(std::move(model_node_before)); - pipeline.push(std::move(model_node_after)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(this->response.mutable_outputs()->count(pipelineOutputName), 1); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_shape()->dim_size(), 2); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_shape()->dim(0).size(), 5); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_shape()->dim(1).size(), 0); - ASSERT_EQ(this->response.mutable_outputs()->at(pipelineOutputName).mutable_tensor_content()->size(), 0); - ASSERT_EQ(this->response.mutable_outputs()->count(outputNameFromCustomNode), 1); - ASSERT_EQ(this->response.mutable_outputs()->at(outputNameFromCustomNode).mutable_tensor_shape()->dim_size(), 2); - ASSERT_EQ(this->response.mutable_outputs()->at(outputNameFromCustomNode).mutable_tensor_shape()->dim(0).size(), 5); - ASSERT_EQ(this->response.mutable_outputs()->at(outputNameFromCustomNode).mutable_tensor_shape()->dim(1).size(), 0); - ASSERT_EQ(this->response.mutable_outputs()->at(outputNameFromCustomNode).mutable_tensor_content()->size(), 0); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, DemultiplexerCreatesShardedFP64TensorsFromEntryNode) { - /* - Description: - - Entry (2x1x2x1x2, fp64) --- demultiplexer --------> (1x2x1x2, fp64) 2x ModelNode (1x2x1x2, fp64) -------> (1x2x1x2, fp64) 2x ModelNode (1x2x1x2, fp64) ----- gather -----> (2x1x2x1x2, fp64) Exit - */ - - // Prepare request - const std::vector inputValues{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}; - PredictRequest request; - PredictResponse response; - tensorflow::TensorProto& proto = (*request.mutable_inputs())[pipelineInputName]; - proto.set_dtype(tensorflow::DataType::DT_DOUBLE); - proto.mutable_tensor_content()->assign((char*)inputValues.data(), inputValues.size() * sizeof(double)); - proto.mutable_tensor_shape()->add_dim()->set_size(2); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(2); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(2); - - // Prepare model - ConstructorEnabledModelManager manager; - ModelConfig config = DUMMY_FP64_MODEL_CONFIG; - config.setBatchingParams(""); - ASSERT_EQ(config.parseShapeParameter("(1,2,1,2)"), ovms::StatusCode::OK); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - - // Prepare pipeline - std::optional demultiplyCount = -1; - std::set gather = {"request"}; - - auto inputTensorInfo = std::make_shared(pipelineOutputName, - Precision::FP64, - Shape{Dimension::any(), 1, 2, 1, 2}); - const tensor_map_t inputsInfo{{pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo, demultiplyCount); - auto tensorInfo = std::make_shared(pipelineOutputName, - Precision::FP64, - Shape{Dimension::any(), 1, 2, 1, 2}); - const tensor_map_t outputsInfo{{pipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo, gather); - auto model_node_1 = std::make_unique("increment_node_1", "dummy_fp64", std::nullopt, manager); - auto model_node_2 = std::make_unique("increment_node_2", "dummy_fp64", std::nullopt, manager); - - auto pipeline = std::make_unique(*input_node, *output_node, *this->reporter); - pipeline->connect(*input_node, *model_node_1, {{pipelineInputName, "input:0"}}); - pipeline->connect(*model_node_1, *model_node_2, {{"output:0", "input:0"}}); - pipeline->connect(*model_node_2, *output_node, {{"output:0", pipelineOutputName}}); - - pipeline->push(std::move(input_node)); - pipeline->push(std::move(model_node_1)); - pipeline->push(std::move(model_node_2)); - pipeline->push(std::move(output_node)); - - // Execute - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - checkIncrement4DimResponse(pipelineOutputName, {3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}, response, {2, 1, 2, 1, 2}); -} - -struct LibraryCountDeinitialize { - inline static int deinitializeCounter; - - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - deinitializeCounter += 1; - return 0; - } - static int execute(const struct CustomNodeTensor* inputs, int, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void* customNodeLibraryInternalManager) { - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, MultipleDeinitializeCallsOnRetire) { - // Nodes - // request custom custom_2 custom_3 response - // O--------->O--------->O--------->O---------->O - // add-sub add-sub add-sub - ResourcesAccessModelManager manager; - ovms::FunctorResourcesCleaner cleaner(manager); - ASSERT_EQ(manager.getResourcesSize(), 0); - PipelineFactory factory; - - // mocking custom node library and copying crucial functions from add_sub_lib in order to - // create pipeline definition - auto mockedLibrary = createLibraryMock(); - mockedLibrary.getInputsInfo = library.getInputsInfo; - mockedLibrary.getOutputsInfo = library.getOutputsInfo; - - // setting global deinitialize call counter to 0 - LibraryCountDeinitialize::deinitializeCounter = 0; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, mockedLibrary, parameters_t{}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, mockedLibrary, parameters_t{}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, mockedLibrary, parameters_t{}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (pipelineInputName) O--------->O custom node (customNodeInputName) - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, customNodeInputName}}}}; - - // custom node (customNodeOutputName) O--------->O custom node 2 (customNodeInputName) - connections["custom_node_2"] = { - {"custom_node", {{customNodeOutputName, customNodeInputName}}}}; - - // custom node 2 (customNodeOutputName) O--------->O custom node 3 (customNodeInputName) - connections["custom_node_3"] = { - {"custom_node_2", {{customNodeOutputName, customNodeInputName}}}}; - - // custom node (customNodeOutputName) O--------->O response (pipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"custom_node_3", {{customNodeOutputName, pipelineOutputName}}}}; - - ASSERT_EQ(factory.createDefinition("my_new_pipeline", info, connections, manager, manager, manager), StatusCode::OK); - cleaner.cleanup(); - ASSERT_EQ(manager.getResourcesSize(), 3); - - factory.retireOtherThan({}, manager); - cleaner.cleanup(); - ASSERT_EQ(manager.getResourcesSize(), 0); - manager.join(); - // Each custom node has effectively 1 internalManager initialized, because they use same library instance - // in order to count whether deinitialize has been called expected number of times - ASSERT_EQ(LibraryCountDeinitialize::deinitializeCounter, 3); -} - -TEST_F(EnsembleFlowCustomNodePipelineExecutionTest, ReloadPipelineWithoutNodeDeinitializeAllCustomNodes) { - // Nodes - // request custom custom_2 custom_3 response - // O--------->O--------->O--------->O---------->O - // add-sub add-sub add-sub - ResourcesAccessModelManager manager; - ovms::FunctorResourcesCleaner cleaner(manager); - cleaner.cleanup(); - ASSERT_EQ(manager.getResourcesSize(), 0); - PipelineFactory factory; - - // mocking custom node library and copying crucial functions from add_sub_lib in order to - // create pipeline definition - auto mockedLibrary = createLibraryMock(); - mockedLibrary.getInputsInfo = library.getInputsInfo; - mockedLibrary.getOutputsInfo = library.getOutputsInfo; - - // setting global deinitialize call counter to 0 - LibraryCountDeinitialize::deinitializeCounter = 0; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{pipelineInputName, pipelineInputName}}}, - {NodeKind::CUSTOM, "custom_node", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, mockedLibrary, parameters_t{}}, - {NodeKind::CUSTOM, "custom_node_2", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, mockedLibrary, parameters_t{}}, - {NodeKind::CUSTOM, "custom_node_3", "", std::nullopt, {{customNodeOutputName, customNodeOutputName}}, - std::nullopt, {}, mockedLibrary, parameters_t{}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (pipelineInputName) O--------->O custom node (customNodeInputName) - connections["custom_node"] = { - {ENTRY_NODE_NAME, {{pipelineInputName, customNodeInputName}}}}; - - // custom node (customNodeOutputName) O--------->O custom node 2 (customNodeInputName) - connections["custom_node_2"] = { - {"custom_node", {{customNodeOutputName, customNodeInputName}}}}; - - // custom node 2 (customNodeOutputName) O--------->O custom node 3 (customNodeInputName) - connections["custom_node_3"] = { - {"custom_node_2", {{customNodeOutputName, customNodeInputName}}}}; - - // custom node (customNodeOutputName) O--------->O response (pipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"custom_node_3", {{customNodeOutputName, pipelineOutputName}}}}; - - ASSERT_EQ(factory.createDefinition("my_new_pipeline", info, connections, manager, manager, manager), StatusCode::OK); - cleaner.cleanup(); - ASSERT_EQ(manager.getResourcesSize(), 3); - - // Nodes - // request custom custom_2 response - // O--------->O--------->O---------->O - // add-sub add-sub - info.erase(info.begin() + 3); - connections.erase("custom_node_3"); - connections[EXIT_NODE_NAME] = { - {"custom_node_2", {{customNodeOutputName, pipelineOutputName}}}}; - ASSERT_EQ(factory.reloadDefinition("my_new_pipeline", std::move(info), std::move(connections), manager, manager, manager), StatusCode::OK); - cleaner.cleanup(); - ASSERT_EQ(manager.getResourcesSize(), 2); - // Each custom node has effectively 1 internalManager initialized, because they use same library instance - // in order to count whether deinitialize has been called expected number of times - ASSERT_EQ(LibraryCountDeinitialize::deinitializeCounter, 3); -} - -static constexpr const char* INPUT_TENSOR_NAME = "input_string"; -static constexpr const char* OUTPUT_TENSOR_NAME = "output_string"; - -struct Passthrough_AnyDim_U8 { - static int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { - return 0; - } - static int deinitialize(void* customNodeLibraryInternalManager) { - return 0; - } - static int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - int numberOfDimensions = 2; // default - for (int i = 0; i < paramsCount; i++) { - if (std::strcmp(params[i].key, "num_of_dims") == 0) { - numberOfDimensions = std::stoi(params[i].value); - } - } - // // Inputs reading - const CustomNodeTensor* input = nullptr; - - for (int i = 0; i < inputsCount; i++) { - if (std::strcmp(inputs[i].name, INPUT_TENSOR_NAME) == 0) { - input = &(inputs[i]); - } else { - std::cout << "Unrecognized input: " << inputs[i].name << std::endl; - return 1; - } - } - - // Preparing output tensor - float* buffer = (float*)malloc(inputs[0].dataBytes); - std::memcpy((uint8_t*)buffer, inputs[0].data, inputs[0].dataBytes); - - *outputsCount = 1; - *outputs = (struct CustomNodeTensor*)malloc(*outputsCount * sizeof(CustomNodeTensor)); - if ((*outputs) == nullptr) { - std::cout << "malloc has failed" << std::endl; - free(buffer); - return 1; - } - - CustomNodeTensor& output = (*outputs)[0]; - output.name = OUTPUT_TENSOR_NAME; - output.data = reinterpret_cast(buffer); - output.dataBytes = inputs[0].dataBytes; - output.dimsCount = numberOfDimensions; - output.dims = (uint64_t*)malloc(output.dimsCount * sizeof(uint64_t)); - for (int i = 0; i < numberOfDimensions; i++) { - output.dims[i] = input->dims[i]; - } - output.precision = U8; - - return 0; - } - static int getInputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - int numberOfDimensions = 2; // default - for (int i = 0; i < paramsCount; i++) { - if (std::strcmp(params[i].key, "num_of_dims") == 0) { - numberOfDimensions = std::stoi(params[i].value); - } - } - - *infoCount = 1; - *info = (struct CustomNodeTensorInfo*)malloc(*infoCount * sizeof(struct CustomNodeTensorInfo)); - - (*info)[0].name = INPUT_TENSOR_NAME; - (*info)[0].dimsCount = numberOfDimensions; - (*info)[0].dims = (uint64_t*)malloc((*info)[0].dimsCount * sizeof(uint64_t)); - for (int i = 0; i < numberOfDimensions; i++) { - (*info)[0].dims[i] = -1; - } - (*info)[0].precision = U8; - - return 0; - } - static int getOutputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager) { - int numberOfDimensions = 2; // default - for (int i = 0; i < paramsCount; i++) { - if (std::strcmp(params[i].key, "num_of_dims") == 0) { - numberOfDimensions = std::stoi(params[i].value); - } - } - - *infoCount = 1; - *info = (struct CustomNodeTensorInfo*)malloc(*infoCount * sizeof(struct CustomNodeTensorInfo)); - - (*info)[0].name = OUTPUT_TENSOR_NAME; - (*info)[0].dimsCount = numberOfDimensions; - (*info)[0].dims = (uint64_t*)malloc((*info)->dimsCount * sizeof(uint64_t)); - for (int i = 0; i < numberOfDimensions; i++) { - (*info)[0].dims[i] = -1; - } - - (*info)[0].precision = U8; - - return 0; - } - static int release(void* ptr, void* customNodeLibraryInternalManager) { - free(ptr); - return 0; - } -}; - -template -class EnsembleFlowStringInput : public ::testing::Test { -public: - void SetUp() override { - } - - RequestType request; - ResponseType response; - std::unique_ptr reporter; - - const std::string customNodeName = "passthrough"; - static constexpr const char* pipelineInputName = "pipeline_input"; - const std::string pipelineOutputName = "pipeline_output"; - const std::string pipelineName = "my_pipeline"; - std::set gatherFromNode = {}; -}; - -using MyTypes = ::testing::Types; -TYPED_TEST_SUITE(EnsembleFlowStringInput, MyTypes); - -TYPED_TEST(EnsembleFlowStringInput, positive_2d) { - // Most basic configuration, just process single passthrough custom node pipeline request - // input passthrough output - // O------->O------->O - std::vector expectedStrings = {"String_123", "zebra", ""}; - prepareInferStringRequest(this->request, this->pipelineInputName, expectedStrings); - - auto inputTensorInfo = std::make_shared(this->pipelineInputName, - ovms::Precision::U8, - ovms::Shape{-1, -1}, - Layout{"NC"}); - const tensor_map_t inputsInfo{{this->pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&this->request, inputsInfo); - auto tensorInfo = std::make_shared(this->pipelineOutputName, - ovms::Precision::U8, - ovms::Shape{-1, -1}, - Layout{"NC"}); - const tensor_map_t outputsInfo{{this->pipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&this->response, outputsInfo, this->gatherFromNode, false, this->pipelineName); - auto mockedLibrary = createLibraryMock(); - auto custom_node = std::make_unique(this->customNodeName, mockedLibrary, parameters_t{}); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *custom_node, {{this->pipelineInputName, INPUT_TENSOR_NAME}}); - pipeline.connect(*custom_node, *output_node, {{OUTPUT_TENSOR_NAME, this->pipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(custom_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - std::vector expectedData = { - 'S', 't', 'r', 'i', 'n', 'g', '_', '1', '2', '3', 0, - 'z', 'e', 'b', 'r', 'a', 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - std::vector expectedShape = {3, 11}; - bool checkRaw = false; - checkIncrement4DimResponse(this->pipelineOutputName, expectedData, this->response, expectedShape, checkRaw); -} - -// Legacy, supported via Native OV String since 2024.0 -TYPED_TEST(EnsembleFlowStringInput, positive_1d) { - // Most basic configuration, just process single passthrough custom node pipeline request - // input passthrough output - // O------->O------->O - std::vector expectedStrings = {"ala", "", "ma", "kota"}; - prepareInferStringRequest(this->request, this->pipelineInputName, expectedStrings); - - auto inputTensorInfo = std::make_shared(this->pipelineInputName, - ovms::Precision::U8, - ovms::Shape{-1}, - Layout{"NC"}); - const tensor_map_t inputsInfo{{this->pipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&this->request, inputsInfo); - auto tensorInfo = std::make_shared(this->pipelineOutputName, - ovms::Precision::U8, - ovms::Shape{-1}, - Layout{"NC"}); - const tensor_map_t outputsInfo{{this->pipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&this->response, outputsInfo, this->gatherFromNode, false, this->pipelineName); - auto mockedLibrary = createLibraryMock(); - auto custom_node = std::make_unique(this->customNodeName, mockedLibrary, parameters_t{{"num_of_dims", "1"}}); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *custom_node, {{this->pipelineInputName, INPUT_TENSOR_NAME}}); - pipeline.connect(*custom_node, *output_node, {{OUTPUT_TENSOR_NAME, this->pipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(custom_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::NOT_IMPLEMENTED); -} diff --git a/src/test/ensemble_tests.cpp b/src/test/ensemble_tests.cpp deleted file mode 100644 index 9e9eec67cd..0000000000 --- a/src/test/ensemble_tests.cpp +++ /dev/null @@ -1,6376 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include -#include - -#include -#include -#include - -#include "../tfs_frontend/tfs_request_utils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" -#include "../kfs_frontend/kfs_utils.hpp" - -#include "../deserialization_main.hpp" - -#include "../dags/dl_node.hpp" -#include "../dags/entry_node.hpp" -#include "../dags/exit_node.hpp" -#include "../dags/nodestreamidguard.hpp" -#include "../dags/pipeline.hpp" -#include "../dags/pipeline_factory.hpp" -#include "../dags/pipelinedefinition.hpp" -#include "../tfs_frontend/deserialization.hpp" -#include "../inference_executor.hpp" -#include "src/filesystem/localfilesystem.hpp" -#include "../logging.hpp" -#if (MEDIAPIPE_DISABLE == 0) -#include "../mediapipe_internal/mediapipefactory.hpp" -#endif -#include "src/metrics/metric_config.hpp" -#include "src/metrics/metric_registry.hpp" -#include "../model.hpp" -#include "../model_metric_reporter.hpp" -#include "../modelconfig.hpp" -#include "../modelinstance.hpp" -#include "src/status.hpp" -#include "../tensor_conversion.hpp" -#include "../timer.hpp" - -#include "constructor_enabled_model_manager.hpp" -#include "platform_utils.hpp" -#include "test_models_configs.hpp" -#include "test_utils.hpp" -#include "light_test_utils.hpp" -#include "test_with_temp_dir.hpp" - -using namespace ovms; -using namespace tensorflow; -using namespace tensorflow::serving; - -using testing::_; -using testing::Return; - -using ::testing::ElementsAre; - -const uint32_t NIREQ = 2; - -template -class EnsembleFlowBothApiTest : public TestWithTempDir { -public: - void SetUp() override { - TestWithTempDir::SetUp(); - // Prepare manager - config = DUMMY_MODEL_CONFIG; - config.setNireq(NIREQ); - - reporter = std::make_unique(&this->metricConfig, &this->registry, "example_pipeline_name", 1); - - // Prepare request - prepareRequest(bs1requestData, request, customPipelineInputName); - requestData = bs1requestData; - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - } - - void prepareRequest(const std::vector& requestData, TFSRequestType& request, const std::string& customPipelineInputName, const ovms::signed_shape_t& shape = {1, DUMMY_MODEL_INPUT_SIZE}) { - request.Clear(); - preparePredictRequest(request, inputs_info_t{{customPipelineInputName, {shape, ovms::Precision::FP32}}}, requestData); - } - - void prepareRequest(const std::vector& requestData, KFSRequest& request, const std::string& customPipelineInputName, const ovms::signed_shape_t& shape = {1, DUMMY_MODEL_INPUT_SIZE}) { - request.Clear(); - prepareKFSInferInputTensor(request, customPipelineInputName, std::make_tuple(shape, ovmsPrecisionToKFSPrecision(ovms::Precision::FP32)), requestData); - } - - void checkDummyResponse(int seriesLength, int batchSize = 1, const std::string& servableName = "") { - ::checkDummyResponse(customPipelineOutputName, requestData, request, response, seriesLength, batchSize, servableName); - } - - void checkScalarResponse(float inputScalar, const std::string& pipelineName) { - ::checkScalarResponse(customPipelineOutputName, inputScalar, response, pipelineName); - } - - void checkStringResponse(const std::vector& inputStrings, const std::string& pipelineName) { - ::checkStringResponse(customPipelineOutputName, inputStrings, response, pipelineName); - } - - ModelConfig config; - RequestType request; - ResponseType response; - MetricRegistry registry; - MetricConfig metricConfig; - std::unique_ptr reporter; - - std::string dummyModelName = "dummy"; - std::optional requestedModelVersion{std::nullopt}; - const std::string customPipelineInputName = "custom_dummy_input"; - const std::string customPipelineOutputName = "custom_dummy_output"; - std::shared_ptr dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - std::shared_ptr dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - - std::vector requestData; - const std::vector bs1requestData{-5.0, 3.0, 0.0, -12.0, 9.0, -100.0, 102.0, 92.0, -1.0, 12.0}; -}; - -// New test suite. Tests both - TFS and KFS. -// Currently only tests which contain gather in exit node. -using MyTypes = ::testing::Types; -TYPED_TEST_SUITE(EnsembleFlowBothApiTest, MyTypes); - -// Old test suite. Tests only TFS API. -class EnsembleFlowTest : public TestWithTempDir { -protected: - void SetUp() override { - TestWithTempDir::SetUp(); - // Prepare manager - config = DUMMY_MODEL_CONFIG; - config.setNireq(NIREQ); - - reporter = std::make_unique(&this->metricConfig, &this->registry, "example_pipeline_name", 1); - - // Prepare request - prepareRequest(bs1requestData, request, customPipelineInputName); - requestData = bs1requestData; - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - } - - void prepareRequest(const std::vector& requestData, PredictRequest& request, const std::string& customPipelineInputName) { - request.Clear(); - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - proto.set_dtype(tensorflow::DataType::DT_FLOAT); - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(DUMMY_MODEL_INPUT_SIZE); - } - - void prepareRequest(const std::vector& requestData, PredictRequest& request, const std::string& customPipelineInputName, const std::vector& shape) { - request.Clear(); - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - proto.set_dtype(tensorflow::DataType::DT_FLOAT); - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - for (size_t i = 0; i < shape.size(); i++) { - proto.mutable_tensor_shape()->add_dim()->set_size(shape[i]); - } - } - - void prepareBinaryRequest(const std::string& jpegPath, PredictRequest& request, const std::string& customPipelineInputName, int batchSize = 1) { - size_t filesize; - std::unique_ptr image_bytes; - readFile(jpegPath, filesize, image_bytes); - - request.Clear(); - tensorflow::TensorProto& inputProto = (*request.mutable_inputs())[customPipelineInputName]; - inputProto.set_dtype(tensorflow::DataType::DT_STRING); - for (int i = 0; i < batchSize; i++) { - inputProto.add_string_val(image_bytes.get(), filesize); - } - inputProto.mutable_tensor_shape()->add_dim()->set_size(batchSize); - } - - void prepareMisalignedBinaryImageRequest(const std::string& image1, const std::string& image2, PredictRequest& request, const std::string& customPipelineInputName) { - request.Clear(); - tensorflow::TensorProto& inputProto = (*request.mutable_inputs())[customPipelineInputName]; - inputProto.set_dtype(tensorflow::DataType::DT_STRING); - - size_t filesize; - std::unique_ptr image_bytes; - readFile(image1, filesize, image_bytes); - inputProto.add_string_val(image_bytes.get(), filesize); - - readFile(image2, filesize, image_bytes); - inputProto.add_string_val(image_bytes.get(), filesize); - - inputProto.mutable_tensor_shape()->add_dim()->set_size(2); - } - - void checkDummyResponse(int seriesLength, int batchSize = 1) { - ::checkDummyResponse(customPipelineOutputName, requestData, request, response, seriesLength, batchSize); - } - - void performWrongPipelineConfigTest(const char* configFileContent) { - std::string fileToReload = directoryPath + "/ovms_config_file1.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(configFileContent), fileToReload); - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.loadConfig(fileToReload); - std::unique_ptr pipeline; - auto status = managerWithDummyModel.getPipelineFactory().create(pipeline, - "pipeline1Dummy", - &request, - &response, - managerWithDummyModel); - ASSERT_EQ(status, ovms::StatusCode::PIPELINE_DEFINITION_NAME_MISSING) << status.string(); - } - - ModelConfig config; - - PredictRequest request; - PredictResponse response; - MetricRegistry registry; - MetricConfig metricConfig; - std::unique_ptr reporter; - - std::string dummyModelName = "dummy"; - std::optional requestedModelVersion{std::nullopt}; - const std::string customPipelineInputName = "custom_dummy_input"; - const std::string customPipelineOutputName = "custom_dummy_output"; - std::shared_ptr dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - std::shared_ptr dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - - std::vector requestData; - const std::vector bs1requestData{-5.0, 3.0, 0.0, -12.0, 9.0, -100.0, 102.0, 92.0, -1.0, 12.0}; -}; - -TYPED_TEST(EnsembleFlowBothApiTest, DummyModel) { - // Most basic configuration, just process single dummy model request - // input dummy output - // O------->O------->O - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(this->config); - - // Configure pipeline - const tensor_map_t inputsInfo{{this->customPipelineInputName, this->dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&this->request, inputsInfo); - auto model_node = std::make_unique("dummy_node", this->dummyModelName, this->requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{this->customPipelineOutputName, this->dagDummyModelOutputTensorInfo}}; - std::set gatherFromNode = {}; - std::string pipelineName = "test_pipeline"; - auto output_node = std::make_unique>(&this->response, outputsInfo, gatherFromNode, true, pipelineName); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node, {{this->customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, this->customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - const int dummySeriallyConnectedCount = 1; - this->checkDummyResponse(dummySeriallyConnectedCount, 1, pipelineName); -} - -TYPED_TEST(EnsembleFlowBothApiTest, NativeStringModel) { - // Most basic configuration, just process single passthrough string model request - // input passthrough output - // O---------->O---------->O - ConstructorEnabledModelManager managerWithStringModel; - this->config = NATIVE_STRING_MODEL_CONFIG; - this->config.setBatchingParams(""); - ASSERT_EQ(managerWithStringModel.reloadModelWithVersions(this->config), ovms::StatusCode::OK_RELOADED); - - // Configure pipeline - this->dagDummyModelInputTensorInfo = std::make_shared(this->customPipelineInputName, - ovms::Precision::STRING, - ovms::Shape{-1}, - Layout{"N..."}); - this->dagDummyModelOutputTensorInfo = std::make_shared(this->customPipelineOutputName, - ovms::Precision::STRING, - ovms::Shape{-1}, - Layout{"N..."}); - const tensor_map_t inputsInfo{{this->customPipelineInputName, this->dagDummyModelInputTensorInfo}}; - std::vector inputStrings = {"ala", "", "ma", "kota"}; - this->request.Clear(); - prepareInferStringRequest(this->request, this->customPipelineInputName, inputStrings); - auto input_node = std::make_unique>(&this->request, inputsInfo); - auto model_node = std::make_unique("string_node", "passthrough_string", this->requestedModelVersion, managerWithStringModel); - const tensor_map_t outputsInfo{{this->customPipelineOutputName, this->dagDummyModelOutputTensorInfo}}; - std::set gatherFromNode = {}; - std::string pipelineName = "test_pipeline"; - auto output_node = std::make_unique>(&this->response, outputsInfo, gatherFromNode, true, pipelineName); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node, {{this->customPipelineInputName, PASSTHROUGH_STRING_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, this->customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - this->checkStringResponse(inputStrings, pipelineName); -} - -TYPED_TEST(EnsembleFlowBothApiTest, ScalarModel) { - // Most basic configuration, just process single scalar model request - // input scalar output - // O------->O------->O - ConstructorEnabledModelManager managerWithScalarModel; - this->config = SCALAR_MODEL_CONFIG; - managerWithScalarModel.reloadModelWithVersions(this->config); - - float inputData = 5.4f; - this->prepareRequest(std::vector{inputData}, this->request, this->customPipelineInputName, ovms::signed_shape_t{}); - - // Configure pipeline - const tensor_map_t inputsInfo{{this->customPipelineInputName, - std::make_shared( - this->customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{}, Layout{"..."})}}; - auto input_node = std::make_unique>(&this->request, inputsInfo); - auto model_node = std::make_unique("scalar_node", "scalar", this->requestedModelVersion, managerWithScalarModel); - const tensor_map_t outputsInfo{{this->customPipelineOutputName, - std::make_shared( - this->customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{}, Layout{"..."})}}; - std::set gatherFromNode = {}; - std::string pipelineName = "test_pipeline"; - auto output_node = std::make_unique>(&this->response, outputsInfo, gatherFromNode, true, pipelineName); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node, {{this->customPipelineInputName, SCALAR_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{SCALAR_MODEL_OUTPUT_NAME, this->customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - this->checkScalarResponse(inputData, pipelineName); -} - -TYPED_TEST(EnsembleFlowBothApiTest, SequenceOfDynamicDummyInferZeroBatch) { - // input 3x dynamic dummy output - // ===[0,10]===>O----------->O->O->O-------->O=====[0,10]===> - ConstructorEnabledModelManager managerWithDummyModel; - this->config = DUMMY_MODEL_CONFIG; - this->config.setBatchingParams("-1"); - managerWithDummyModel.reloadModelWithVersions(this->config); - int batchSize = 0; - - std::vector inputData; // no data (0,10) - this->prepareRequest(inputData, this->request, this->customPipelineInputName, ovms::signed_shape_t{batchSize, 10}); - - // Configure pipeline - const tensor_map_t inputsInfo{{this->customPipelineInputName, - std::make_shared( - this->customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{-1, 10}, Layout{"..."})}}; - auto input_node = std::make_unique>(&this->request, inputsInfo); - auto model_node1 = std::make_unique("dummy_node_1", "dummy", this->requestedModelVersion, managerWithDummyModel); - auto model_node2 = std::make_unique("dummy_node_2", "dummy", this->requestedModelVersion, managerWithDummyModel); - auto model_node3 = std::make_unique("dummy_node_3", "dummy", this->requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{this->customPipelineOutputName, - std::make_shared( - this->customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{-1, 10}, Layout{"..."})}}; - std::set gatherFromNode = {}; - std::string pipelineName = "test_pipeline"; - auto output_node = std::make_unique>(&this->response, outputsInfo, gatherFromNode, true, pipelineName); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node1, {{this->customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node1, *model_node2, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node2, *model_node3, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node3, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, this->customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node1)); - pipeline.push(std::move(model_node2)); - pipeline.push(std::move(model_node3)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - int series = 3; - this->checkDummyResponse(series, batchSize, pipelineName); -} - -TYPED_TEST(EnsembleFlowBothApiTest, TwoInnerNodesConnectedShapeRangePartiallyMatching) { - ConstructorEnabledModelManager managerWithDummyModel; - - this->config = DUMMY_MODEL_CONFIG; - this->config.setName("dummy_A"); - this->config.setBatchSize(std::nullopt); - this->config.parseShapeParameter("(-1,1:3)"); - managerWithDummyModel.reloadModelWithVersions(this->config); - - this->config = DUMMY_MODEL_CONFIG; - this->config.setName("dummy_B"); - this->config.setBatchSize(std::nullopt); - this->config.parseShapeParameter("(-1,2:4)"); - managerWithDummyModel.reloadModelWithVersions(this->config); - - // Configure pipeline - this->dagDummyModelOutputTensorInfo = std::make_shared(this->customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{Dimension::any(), {1, 3}}, - Layout{"NC"}); - this->dagDummyModelInputTensorInfo = std::make_shared(this->customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{Dimension::any(), {2, 4}}, - Layout{"NC"}); - - // 2x2 passing - { - this->prepareRequest(std::vector{5.0, 6.0, 15.0, 16.0}, this->request, this->customPipelineInputName, {2, 2}); - this->response.Clear(); - - const tensor_map_t inputsInfo{{this->customPipelineInputName, this->dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&this->request, inputsInfo); - auto model_node_A = std::make_unique("dummy_node_A", "dummy_A", this->requestedModelVersion, managerWithDummyModel); - auto model_node_B = std::make_unique("dummy_node_B", "dummy_B", this->requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{this->customPipelineOutputName, this->dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&this->response, outputsInfo); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node_A, {{this->customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_A, *model_node_B, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_B, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, this->customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node_A)); - pipeline.push(std::move(model_node_B)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse(this->customPipelineOutputName, std::vector{7.0, 8.0, 17.0, 18.0}, this->response, {2, 2}); - } - - // 2x4 not passing due to not matched dummy_A (but matching dummy_B) - { - this->prepareRequest(std::vector{5.0, 6.0, 15.0, 16.0, 5.0, 6.0, 15.0, 16.0}, this->request, this->customPipelineInputName, {2, 4}); - this->response.Clear(); - - const tensor_map_t inputsInfo{{this->customPipelineInputName, this->dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&this->request, inputsInfo); - auto model_node_A = std::make_unique("dummy_node_A", "dummy_A", this->requestedModelVersion, managerWithDummyModel); - auto model_node_B = std::make_unique("dummy_node_B", "dummy_B", this->requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{this->customPipelineOutputName, this->dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&this->response, outputsInfo); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node_A, {{this->customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_A, *model_node_B, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_B, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, this->customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node_A)); - pipeline.push(std::move(model_node_B)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); - } - - // 2x1 not passing due to not matched dummy_B (but matching dummy_A) - { - this->prepareRequest(std::vector{5.0, 6.0}, this->request, this->customPipelineInputName, {2, 1}); - this->response.Clear(); - - const tensor_map_t inputsInfo{{this->customPipelineInputName, this->dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&this->request, inputsInfo); - auto model_node_A = std::make_unique("dummy_node_A", "dummy_A", this->requestedModelVersion, managerWithDummyModel); - auto model_node_B = std::make_unique("dummy_node_B", "dummy_B", this->requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{this->customPipelineOutputName, this->dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&this->response, outputsInfo); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node_A, {{this->customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_A, *model_node_B, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_B, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, this->customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node_A)); - pipeline.push(std::move(model_node_B)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); - } -} - -// This test is only theoretical scenario, since pipeline validation should not allow such pipelines. -TEST_F(EnsembleFlowTest, TwoInnerNodesConnectedShapeRangeNotMatching) { - ConstructorEnabledModelManager managerWithDummyModel; - - config = DUMMY_MODEL_CONFIG; - config.setName("dummy_A"); - config.setBatchSize(std::nullopt); - config.parseShapeParameter("(-1,1:3)"); - managerWithDummyModel.reloadModelWithVersions(config); - - config = DUMMY_MODEL_CONFIG; - config.setName("dummy_B"); - config.setBatchSize(std::nullopt); - config.parseShapeParameter("(-1,4:6)"); - managerWithDummyModel.reloadModelWithVersions(config); - - // Configure pipeline - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{Dimension::any(), {1, 3}}, - Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{Dimension::any(), {4, 6}}, - Layout{"NC"}); - - // 2x2 not matching dummy_B at execution time - prepareRequest(std::vector{5.0, 6.0, 15.0, 16.0}, request, customPipelineInputName, {2, 2}); - response.Clear(); - - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node_A = std::make_unique("dummy_node_A", "dummy_A", requestedModelVersion, managerWithDummyModel); - auto model_node_B = std::make_unique("dummy_node_B", "dummy_B", requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node_A, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_A, *model_node_B, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_B, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node_A)); - pipeline.push(std::move(model_node_B)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); -} - -class EnsembleFlowValidationTest : public EnsembleFlowTest { -public: - std::unique_ptr createDummyPipeline(ConstructorEnabledModelManager& managerWithDummyModel) { - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - auto pipeline = std::make_unique(*input_node, *output_node, *this->reporter); - pipeline->connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline->connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline->push(std::move(input_node)); - pipeline->push(std::move(model_node)); - pipeline->push(std::move(output_node)); - return pipeline; - } -}; - -TEST_F(EnsembleFlowValidationTest, DummyModelValid) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorNumberOfInputs) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())["input1"]; - auto& proto2 = (*request.mutable_inputs())["input2"]; - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_NO_OF_INPUTS); - proto1.Clear(); - proto2.Clear(); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorMissingInput) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())["input1"]; - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_MISSING_INPUT); - proto1.Clear(); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorShapeValueNegative) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.mutable_tensor_shape()->add_dim()->set_size(1); - proto1.mutable_tensor_shape()->add_dim()->set_size(-10); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorBinaryInputWrongNumberOfShapeDimensions) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.set_dtype(tensorflow::DataType::DT_STRING); - proto1.mutable_tensor_shape()->add_dim()->set_size(1); - proto1.mutable_tensor_shape()->add_dim()->set_size(1); - - // enforce the endpoint to be 4d to not fall into string handling - this->dagDummyModelInputTensorInfo = std::make_shared(this->customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{1, 224, 224, 3}, - ovms::Layout{"NHWC"}); - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorBinaryInputBatchSizeMismatch) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.set_dtype(tensorflow::DataType::DT_STRING); - proto1.mutable_tensor_shape()->add_dim()->set_size(2); - - // enforce the endpoint to be 4d to not fall into string handling - this->dagDummyModelInputTensorInfo = std::make_shared(this->customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{1, 224, 224, 3}, - ovms::Layout{"NHWC"}); - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorPrecisionMismatch) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.mutable_tensor_shape()->add_dim()->set_size(1); - proto1.mutable_tensor_shape()->add_dim()->set_size(10); - proto1.set_dtype(tensorflow::DataType::DT_INT32); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_PRECISION); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorInvalidNumberOfShapeDimensions) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.mutable_tensor_shape()->add_dim()->set_size(1); - proto1.mutable_tensor_shape()->add_dim()->set_size(10); - proto1.mutable_tensor_shape()->add_dim()->set_size(3); - proto1.set_dtype(tensorflow::DataType::DT_FLOAT); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorInvalidBatchSize) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.mutable_tensor_shape()->add_dim()->set_size(2); - proto1.mutable_tensor_shape()->add_dim()->set_size(10); - proto1.set_dtype(tensorflow::DataType::DT_FLOAT); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorInvalidShape) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.mutable_tensor_shape()->add_dim()->set_size(1); - proto1.mutable_tensor_shape()->add_dim()->set_size(11); - proto1.set_dtype(tensorflow::DataType::DT_FLOAT); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleFlowValidationTest, DummyModelProtoValidationErrorInvalidTensorContentSize) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.mutable_tensor_shape()->add_dim()->set_size(1); - proto1.mutable_tensor_shape()->add_dim()->set_size(10); - proto1.set_dtype(tensorflow::DataType::DT_FLOAT); - const std::vector data{1.0f}; - proto1.mutable_tensor_content()->assign((char*)data.data(), data.size() * sizeof(float)); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_CONTENT_SIZE); -} - -class EnsembleFlowValidationShapeRangeTest : public EnsembleFlowValidationTest { -protected: - void SetUp() { - EnsembleFlowValidationTest::SetUp(); - - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{{1, 10}, {2, 11}}, - Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{{1, 10}, {2, 11}}, - Layout{"NC"}); - - config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(1:10,2:11)"); - } -}; - -TEST_F(EnsembleFlowValidationShapeRangeTest, DummyModelValid) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); -} - -TEST_F(EnsembleFlowValidationShapeRangeTest, DummyModelProtoValidationErrorInvalidBatchSize) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.mutable_tensor_shape()->add_dim()->set_size(11); - proto1.mutable_tensor_shape()->add_dim()->set_size(10); - proto1.set_dtype(tensorflow::DataType::DT_FLOAT); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(EnsembleFlowValidationShapeRangeTest, DummyModelProtoValidationErrorInvalidShape) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - request.Clear(); - auto& proto1 = (*request.mutable_inputs())[customPipelineInputName]; - proto1.mutable_tensor_shape()->add_dim()->set_size(6); - proto1.mutable_tensor_shape()->add_dim()->set_size(1); - proto1.set_dtype(tensorflow::DataType::DT_FLOAT); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); -} - -class EnsembleFlowValidationShapeAnyTest : public EnsembleFlowValidationTest { -protected: - void SetUp() { - EnsembleFlowValidationTest::SetUp(); - - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), ovms::Dimension::any()}, - Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), ovms::Dimension::any()}, - Layout{"NC"}); - - config = DUMMY_MODEL_CONFIG; - config.setBatchingParams(""); - config.parseShapeParameter("(-1,-1)"); - } -}; - -TEST_F(EnsembleFlowValidationShapeAnyTest, DummyModelValid) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - auto pipeline = createDummyPipeline(managerWithDummyModel); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); -} - -TEST_F(EnsembleFlowTest, DummyModelDirectAndPipelineInference) { - ConstructorEnabledModelManager managerWithDummyModel; - config.setNireq(1); - managerWithDummyModel.reloadModelWithVersions(config); - - // Get dummy model instance - std::shared_ptr model; - std::unique_ptr unload_guard; - auto status = managerWithDummyModel.getModelInstance(dummyModelName, 0, model, unload_guard); - ASSERT_EQ(status, ovms::StatusCode::OK); - - // Prepare request for dummy model directly - tensorflow::serving::PredictRequest simpleModelRequest; - preparePredictRequest(simpleModelRequest, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); - std::vector requestData{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; - auto& input = (*simpleModelRequest.mutable_inputs())[DUMMY_MODEL_INPUT_NAME]; - input.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - tensorflow::serving::PredictResponse simpleModelResponse; - // Do the inference directly on dummy model before inference on pipeline - ASSERT_EQ(ovms::infer(*model, &simpleModelRequest, &simpleModelResponse, unload_guard), ovms::StatusCode::OK); - - ASSERT_EQ(simpleModelResponse.outputs().count(DUMMY_MODEL_OUTPUT_NAME), 1); - auto& output_tensor = (*simpleModelResponse.mutable_outputs())[DUMMY_MODEL_OUTPUT_NAME]; - ASSERT_EQ(output_tensor.tensor_shape().dim_size(), 2); - EXPECT_EQ(output_tensor.tensor_shape().dim(0).size(), 1); - EXPECT_EQ(output_tensor.tensor_shape().dim(1).size(), 10); - - std::vector responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [](float& v) { v += 1.0; }); - - float* actual_output = (float*)output_tensor.tensor_content().data(); - float* expected_output = responseData.data(); - const int dataLengthToCheck = DUMMY_MODEL_OUTPUT_SIZE * sizeof(float); - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, dataLengthToCheck)) - << readableError(expected_output, actual_output, dataLengthToCheck); - - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - const int dummySeriallyConnectedCount = 1; - checkDummyResponse(dummySeriallyConnectedCount); - - // Do the inference directly on dummy model after inference on pipeline - ASSERT_EQ(ovms::infer(*model, &simpleModelRequest, &simpleModelResponse, unload_guard), ovms::StatusCode::OK); - - ASSERT_EQ(simpleModelResponse.outputs().count(DUMMY_MODEL_OUTPUT_NAME), 1); - output_tensor = (*simpleModelResponse.mutable_outputs())[DUMMY_MODEL_OUTPUT_NAME]; - ASSERT_EQ(output_tensor.tensor_shape().dim_size(), 2); - EXPECT_EQ(output_tensor.tensor_shape().dim(0).size(), 1); - EXPECT_EQ(output_tensor.tensor_shape().dim(1).size(), 10); - - actual_output = (float*)output_tensor.tensor_content().data(); - expected_output = responseData.data(); - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, dataLengthToCheck)) - << readableError(expected_output, actual_output, dataLengthToCheck); -} - -TEST_F(EnsembleFlowTest, SeriesOfDummyModels) { - // Most basic configuration, just process single dummy model request - - enum : unsigned int { - PREPARE, - EXECUTE, - COMPARE, - TIMER_END - }; - Timer timer; - timer.start(PREPARE); - - const int N = 100; - // input dummy x N output - // O------->O->O...O->O------->O - - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - std::unique_ptr dummy_nodes[N]; - for (int i = 0; i < N; i++) { - dummy_nodes[i] = std::make_unique("dummy_node_" + std::to_string(i), dummyModelName, requestedModelVersion, managerWithDummyModel); - } - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *(dummy_nodes[0]), {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*(dummy_nodes[N - 1]), *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - for (int i = 0; i < N - 1; i++) { - pipeline.connect(*(dummy_nodes[i]), *(dummy_nodes[i + 1]), {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - } - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(output_node)); - for (auto& dummy_node : dummy_nodes) { - pipeline.push(std::move(dummy_node)); - } - - timer.stop(PREPARE); - timer.start(EXECUTE); - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - timer.stop(EXECUTE); - - timer.start(COMPARE); - checkDummyResponse(N); - timer.stop(COMPARE); - - std::cout << "prepare pipeline: " << timer.elapsed(PREPARE) / 1000 << "ms\n"; - std::cout << "pipeline::execute: " << timer.elapsed(EXECUTE) / 1000 << "ms\n"; - std::cout << "compare results: " << timer.elapsed(COMPARE) / 1000 << "ms\n"; -} - -TEST_F(EnsembleFlowTest, ExecutePipelineWithBatchSizeAny) { - // Scenario - - // input(3x10) dummy(1x10), change batch size to any output(3x10) - // O-------------------------->O----------------------------->O - - // input 3x10 - // dummy is natively 1x10, batch size change to -1 (any) - // process dummy - // check if output is 3x10 - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - const int batchSize = 3; - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(batchSize); - requestData = { - -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, // batch 1 - -15, -14, -13, -12, -11, 11, 12, 13, 14, 15, // batch 2 - -25, -24, -23, -22, -21, 21, 22, 23, 24, 25, // batch 3 - }; - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchingParams("-1"); - ConstructorEnabledModelManager managerWithDynamicBatchDummyModel; - managerWithDynamicBatchDummyModel.reloadModelWithVersions(config); - - // Configure pipeline - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, ovms::Precision::FP32, ovms::Shape{ovms::Dimension::any(), 10}, Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, ovms::Precision::FP32, ovms::Shape{ovms::Dimension::any(), 10}, Layout{"NC"}); - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDynamicBatchDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - const int seriallyConnectedDummyModels = 1; - checkDummyResponse(seriallyConnectedDummyModels, batchSize); -} - -TEST_F(EnsembleFlowTest, ExecutePipelineWithBatchSizeRange) { - // Scenario - - // input(3x10) dummy(1x10), change batch size to (1:5x10) output(3x10) - // O-------------------------->O------------------------------->O - - // input 3x10 - // dummy is natively 1x10, batch size change to 1:5 (range) - // process dummy - // check if output is 3x10 - // check if execution fails for batch higher than 5 - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - int batchSize = 3; - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(batchSize); - requestData = { - -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, // batch 1 - -15, -14, -13, -12, -11, 11, 12, 13, 14, 15, // batch 2 - -25, -24, -23, -22, -21, 21, 22, 23, 24, 25, // batch 3 - }; - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchingParams("1:5"); - ConstructorEnabledModelManager managerWithDynamicBatchDummyModel; - managerWithDynamicBatchDummyModel.reloadModelWithVersions(config); - - // Configure pipeline - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, ovms::Precision::FP32, ovms::Shape{{1, 5}, 10}, Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, ovms::Precision::FP32, ovms::Shape{{1, 5}, 10}, Layout{"NC"}); - { - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDynamicBatchDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - const int seriallyConnectedDummyModels = 1; - checkDummyResponse(seriallyConnectedDummyModels, batchSize); - } - // Prepare invalid data - batchSize = 6; - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(batchSize); - requestData = std::vector(batchSize * 10, 1.234); - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - { - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDynamicBatchDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_BATCH_SIZE); - } -} - -TEST_F(EnsembleFlowTest, ExecutePipelineWithShapeAny) { - // Scenario - - // input(1x5) dummy(1x10) second dimension set to any output(1x5) - // O---------------------->O----------------------------------------------->O - - // input 1x5 - // dummy is natively 1x10, but second dimension set to any (-1) - // process dummy - // check if output is 1x5 - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - proto.mutable_tensor_shape()->mutable_dim(1)->set_size(5); - std::vector requestData = { - -5, -4, -3, -2, -1, // batch 1 - }; - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchSize(std::nullopt); // = not specified in --batch_size parameter - config.parseShapeParameter("(1,-1)"); - ConstructorEnabledModelManager managerWithDynamicShapeDummyModel; - managerWithDynamicShapeDummyModel.reloadModelWithVersions(config); - - // Configure pipeline - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, ovms::Precision::FP32, ovms::Shape{1, ovms::Dimension::any()}, Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, ovms::Precision::FP32, ovms::Shape{1, ovms::Dimension::any()}, Layout{"NC"}); - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDynamicShapeDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - ASSERT_EQ(response.outputs().count(customPipelineOutputName), 1); - const auto& output_proto = response.outputs().at(customPipelineOutputName); - - ASSERT_EQ(output_proto.tensor_content().size(), 1 * 5 * sizeof(float)); - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 2); - ASSERT_EQ(output_proto.tensor_shape().dim(0).size(), 1); - ASSERT_EQ(output_proto.tensor_shape().dim(1).size(), 5); - - std::vector responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [](float& v) { v += 1.0; }); - - float* actual_output = (float*)output_proto.tensor_content().data(); - float* expected_output = responseData.data(); - - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, 1 * 5 * sizeof(float))); -} - -TEST_F(EnsembleFlowTest, ExecutePipelineWithShapeRange) { - // Scenario - - // input(1x5) dummy(1x10) second dimension set to range (1:5) output(1x5) - // O---------------------->O----------------------------------------------->O - - // input 1x5 - // dummy is natively 1x10, but second dimension set to (1:5) range - // process dummy - // check if output is 1x5 - // check if there is an error for input dimension higher than 5 - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - proto.mutable_tensor_shape()->mutable_dim(1)->set_size(5); - std::vector requestData = { - -5, -4, -3, -2, -1, // batch 1 - }; - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchSize(std::nullopt); // = not specified in --batch_size parameter - config.parseShapeParameter("(1,1:5)"); - ConstructorEnabledModelManager managerWithDynamicShapeDummyModel; - managerWithDynamicShapeDummyModel.reloadModelWithVersions(config); - - // Configure pipeline - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, ovms::Precision::FP32, ovms::Shape{1, {1, 5}}, Layout{"NC"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, ovms::Precision::FP32, ovms::Shape{1, {1, 5}}, Layout{"NC"}); - { - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDynamicShapeDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - ASSERT_EQ(response.outputs().count(customPipelineOutputName), 1); - const auto& output_proto = response.outputs().at(customPipelineOutputName); - - ASSERT_EQ(output_proto.tensor_content().size(), 1 * 5 * sizeof(float)); - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 2); - ASSERT_EQ(output_proto.tensor_shape().dim(0).size(), 1); - ASSERT_EQ(output_proto.tensor_shape().dim(1).size(), 5); - - std::vector responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [](float& v) { v += 1.0; }); - - float* actual_output = (float*)output_proto.tensor_content().data(); - float* expected_output = responseData.data(); - - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, 1 * 5 * sizeof(float))); - } - // Prepare invalid data - proto.mutable_tensor_shape()->mutable_dim(1)->set_size(6); - requestData = std::vector(6, 1.234); - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - { - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDynamicShapeDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); - } -} - -TEST_F(EnsembleFlowTest, ExecutePipelineWithBatchAndShapeSetToAny) { - // Scenario - - // input(3x500) dummy(1x10), all dimensions set to any output(3x500) - // O------------------------------>O----------------------------->O - - // input 3x500 - // dummy is natively 1x10, but all dimensions set to any - // process dummy - // check if output is 3x500 - - const int BATCH_SIZE = 3; - const int WIDTH = 500; - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(BATCH_SIZE); - proto.mutable_tensor_shape()->mutable_dim(1)->set_size(WIDTH); - std::vector requestData; - for (int i = 0; i < BATCH_SIZE; i++) { - for (int j = 0; j < WIDTH; j++) { - requestData.push_back((i + 1) * (j + 1)); - /* - 1.0, 2.0, 3.0, ..., 500.0, - 2.0, 4.0, 6.0, ..., 1000.0, - 3.0, 6.0, 9.0, ..., 1500.0 - */ - } - } - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchSize(std::nullopt); // simulate --batch_size parameter not set - config.parseShapeParameter("(-1,-1)"); - ConstructorEnabledModelManager manager; - manager.reloadModelWithVersions(config); - - // Configure pipeline - auto inputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), ovms::Dimension::any()}, - Layout{"NC"}); - const tensor_map_t inputsInfo{{customPipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, manager); - auto tensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{ovms::Dimension::any(), ovms::Dimension::any()}, - Layout{"NC"}); - const tensor_map_t outputsInfo{{customPipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - - ASSERT_EQ(response.outputs().count(customPipelineOutputName), 1); - const auto& output_proto = response.outputs().at(customPipelineOutputName); - - ASSERT_EQ(output_proto.tensor_content().size(), BATCH_SIZE * WIDTH * sizeof(float)); - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 2); - ASSERT_EQ(output_proto.tensor_shape().dim(0).size(), BATCH_SIZE); - ASSERT_EQ(output_proto.tensor_shape().dim(1).size(), WIDTH); - - std::vector responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [](float& v) { v += 1.0; }); - - float* actual_output = (float*)output_proto.tensor_content().data(); - float* expected_output = responseData.data(); - - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, BATCH_SIZE * WIDTH * sizeof(float))); -} - -TEST_F(EnsembleFlowTest, ExecutePipelineWithBatchAndShapeSetToRange) { - // Scenario - - // input(3x500) dummy(1x10), all dimensions set to range (1:1000) output(3x500) - // O------------------------------>O----------------------------->O - - // input 3x500 - // dummy is natively 1x10, but all dimensions are reset to support range (1:1000) - // process dummy - // check if output is 3x500 - // check if for input dimension higher than 1000 there is an error - - const int BATCH_SIZE = 3; - const int WIDTH = 500; - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(BATCH_SIZE); - proto.mutable_tensor_shape()->mutable_dim(1)->set_size(WIDTH); - std::vector requestData; - for (int i = 0; i < BATCH_SIZE; i++) { - for (int j = 0; j < WIDTH; j++) { - requestData.push_back((i + 1) * (j + 1)); - /* - 1.0, 2.0, 3.0, ..., 500.0, - 2.0, 4.0, 6.0, ..., 1000.0, - 3.0, 6.0, 9.0, ..., 1500.0 - */ - } - } - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchSize(std::nullopt); // simulate --batch_size parameter not set - config.parseShapeParameter("(1:1000,1:1000)"); - ConstructorEnabledModelManager manager; - manager.reloadModelWithVersions(config); - - // Configure pipeline - auto inputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{{1, 1000}, {1, 1000}}, - Layout{"NC"}); - auto tensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{{1, 1000}, {1, 1000}}, - Layout{"NC"}); - { - const tensor_map_t inputsInfo{{customPipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, manager); - - const tensor_map_t outputsInfo{{customPipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - - ASSERT_EQ(response.outputs().count(customPipelineOutputName), 1); - const auto& output_proto = response.outputs().at(customPipelineOutputName); - - ASSERT_EQ(output_proto.tensor_content().size(), BATCH_SIZE * WIDTH * sizeof(float)); - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 2); - ASSERT_EQ(output_proto.tensor_shape().dim(0).size(), BATCH_SIZE); - ASSERT_EQ(output_proto.tensor_shape().dim(1).size(), WIDTH); - - std::vector responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [](float& v) { v += 1.0; }); - - float* actual_output = (float*)output_proto.tensor_content().data(); - float* expected_output = responseData.data(); - - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, BATCH_SIZE * WIDTH * sizeof(float))); - } - // Prepare invalid data - proto.mutable_tensor_shape()->mutable_dim(1)->set_size(1001); - requestData = std::vector(BATCH_SIZE * 1001, 1.234); - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - { - const tensor_map_t inputsInfo{{customPipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, manager); - - const tensor_map_t outputsInfo{{customPipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::INVALID_SHAPE); - } -} - -// Disabled with deserialization unification. For this use case to work we would have to additionally rely on "isPipeline" in getFinalShapedTensorInfo() to not use shape from tensor info but to rely on tensorProto -TEST_F(EnsembleFlowTest, DISABLED_ExecutePipelineWithDynamicBatchSize) { - // Scenario - - // input(3x10) dummy(1x10), change batch size output(3x10) - // O-------------------------->O----------------------->O - - // input 3x10 - // dummy is 1x10, perform model batch size change to 3x10 - // process dummy - // check if output is 3x10 - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - const int batchSize = 3; - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(batchSize); - requestData = { - -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, // batch 1 - -15, -14, -13, -12, -11, 11, 12, 13, 14, 15, // batch 2 - -25, -24, -23, -22, -21, 21, 22, 23, 24, 25, // batch 3 - }; - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchingParams("auto"); - ConstructorEnabledModelManager managerWithDynamicBatchDummyModel; - managerWithDynamicBatchDummyModel.reloadModelWithVersions(config); - - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDynamicBatchDummyModel); - auto outputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{3, DUMMY_MODEL_OUTPUT_SIZE}, - Layout{"NC"}); - const tensor_map_t outputsInfo{{customPipelineOutputName, outputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - const int seriallyConnectedDummyModels = 1; - checkDummyResponse(seriallyConnectedDummyModels, batchSize); -} - -// Disabled with deserialization unification. For this use case to work we would have to additionally rely on "isPipeline" in getFinalShapedTensorInfo() to not use shape from tensor info but to rely on tensorProto -TEST_F(EnsembleFlowTest, DISABLED_ExecutePipelineWithDynamicShape) { - // Scenario - - // input(1x5) dummy(1x10), reshape output(1x5) - // O---------------------->O--------------------------->O - - // input 1x5 - // dummy is 1x10, perform model reshape to 1x5 - // process dummy - // check if output is 1x5 - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - proto.mutable_tensor_shape()->mutable_dim(1)->set_size(5); - std::vector requestData = { - -5, -4, -3, -2, -1, // batch 1 - }; - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchSize(std::nullopt); // = not specified in --batch_size parameter - config.parseShapeParameter("auto"); - ConstructorEnabledModelManager managerWithDynamicShapeDummyModel; - managerWithDynamicShapeDummyModel.reloadModelWithVersions(config); - - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDynamicShapeDummyModel); - auto tensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{1, 5}, - Layout{"NC"}); - const tensor_map_t outputsInfo{{customPipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - ASSERT_EQ(response.outputs().count(customPipelineOutputName), 1); - const auto& output_proto = response.outputs().at(customPipelineOutputName); - - ASSERT_EQ(output_proto.tensor_content().size(), 1 * 5 * sizeof(float)); - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 2); - ASSERT_EQ(output_proto.tensor_shape().dim(0).size(), 1); - ASSERT_EQ(output_proto.tensor_shape().dim(1).size(), 5); - - std::vector responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [](float& v) { v += 1.0; }); - - float* actual_output = (float*)output_proto.tensor_content().data(); - float* expected_output = responseData.data(); - - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, 1 * 5 * sizeof(float))); -} - -TEST_F(EnsembleFlowTest, DISABLED_ExecutePipelineWithDynamicBatchAndShape) { - // Scenario - - // input(3x500) dummy(1x10), reshape, change batch size output(3x500) - // O------------------------------>O----------------------------->O - - // input 3x500 - // dummy is 1x10, perform model batch size change to 3x500 - // process dummy - // check if output is 3x500 - - const int BATCH_SIZE = 3; - const int WIDTH = 500; - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(BATCH_SIZE); - proto.mutable_tensor_shape()->mutable_dim(1)->set_size(WIDTH); - std::vector requestData; - for (int i = 0; i < BATCH_SIZE; i++) { - for (int j = 0; j < WIDTH; j++) { - requestData.push_back((i + 1) * (j + 1)); - /* - 1.0, 2.0, 3.0, ..., 500.0, - 2.0, 4.0, 6.0, ..., 1000.0, - 3.0, 6.0, 9.0, ..., 1500.0 - */ - } - } - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchSize(std::nullopt); // simulate --batch_size parameter not set - config.parseShapeParameter("auto"); - ConstructorEnabledModelManager manager; - manager.reloadModelWithVersions(config); - - // Configure pipeline - auto inputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{3, 500}, - Layout{"NC"}); - const tensor_map_t inputsInfo{{customPipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, manager); - auto tensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{3, 500}, - Layout{"NC"}); - const tensor_map_t outputsInfo{{customPipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - - ASSERT_EQ(response.outputs().count(customPipelineOutputName), 1); - const auto& output_proto = response.outputs().at(customPipelineOutputName); - - ASSERT_EQ(output_proto.tensor_content().size(), BATCH_SIZE * WIDTH * sizeof(float)); - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 2); - ASSERT_EQ(output_proto.tensor_shape().dim(0).size(), BATCH_SIZE); - ASSERT_EQ(output_proto.tensor_shape().dim(1).size(), WIDTH); - - std::vector responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [](float& v) { v += 1.0; }); - - float* actual_output = (float*)output_proto.tensor_content().data(); - float* expected_output = responseData.data(); - - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, BATCH_SIZE * WIDTH * sizeof(float))); -} - -TEST_F(EnsembleFlowTest, DISABLED_ExecutePipelineWithDynamicShape_RequestHasDifferentDim0) { - // Scenario - // Shape is set to auto but only first dimension differs - change batch size via reshape - - // input(20x10) dummy(1x10), reshape output(20x10) - // O------------------------------>O----------------------------->O - - // input 20x10 - // dummy is 1x10, perform model reshape to 20x10 - // process dummy - // check if output is 20x10 - - const int BATCH_SIZE = 20; - const int WIDTH = 10; - - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(BATCH_SIZE); - proto.mutable_tensor_shape()->mutable_dim(1)->set_size(WIDTH); - requestData.clear(); - for (int i = 0; i < BATCH_SIZE; i++) { - for (int j = 0; j < WIDTH; j++) { - requestData.push_back((i + 1) * (j + 1)); - /* - 1.0, 2.0, 3.0, ..., 10.0, - 2.0, 4.0, 6.0, ..., 20.0, - 3.0, 6.0, 9.0, ..., 30.0, - ... - 20.0, 40.0, ..., 200.0 - */ - } - } - proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); - - config.setBatchSize(std::nullopt); // simulate --batch_size parameter not set - config.parseShapeParameter("auto"); - ConstructorEnabledModelManager manager; - manager.reloadModelWithVersions(config); - - // Configure pipeline - auto inputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - ovms::Shape{BATCH_SIZE, WIDTH}, - Layout{"NC"}); - const tensor_map_t inputsInfo{{customPipelineInputName, inputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, manager); - auto tensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - ovms::Shape{BATCH_SIZE, WIDTH}, - Layout{"NC"}); - const tensor_map_t outputsInfo{{customPipelineOutputName, tensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - - const int seriallyConnectedDummyModels = 1; - checkDummyResponse(seriallyConnectedDummyModels, BATCH_SIZE); -} - -TEST_F(EnsembleFlowTest, ParallelDummyModels) { - // Most basic configuration, just process single dummy model request - const int N = 200; - /* input dummy x N output - O---------->O------------->O - ... ... /\ - L---------->O-------------_| - */ - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - // Configure pipeline - tensor_map_t inputsInfoTmp; - for (int i = 0; i < N; i++) { - const std::string inputName = customPipelineInputName + std::to_string(i); - inputsInfoTmp[inputName] = std::make_shared(inputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - } - const tensor_map_t inputsInfo = inputsInfoTmp; - auto input_node = std::make_unique>(&request, inputsInfo); - tensor_map_t outputsInfo; - for (size_t i = 0; i < N; ++i) { - const std::string outputName = customPipelineOutputName + std::to_string(i); - outputsInfo.emplace(outputName, - std::make_shared(outputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"})); - } - auto output_node = std::make_unique>(&response, outputsInfo); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - std::unique_ptr dummy_nodes[N]; - - for (int i = 0; i < N; i++) { - dummy_nodes[i] = std::make_unique("dummy_node_" + std::to_string(i), dummyModelName, requestedModelVersion, managerWithDummyModel); - pipeline.connect(*input_node, *(dummy_nodes[i]), {{customPipelineInputName + std::to_string(i), DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*(dummy_nodes[i]), *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName + std::to_string(i)}}); - pipeline.push(std::move(dummy_nodes[i])); - } - pipeline.push(std::move(input_node)); - pipeline.push(std::move(output_node)); - - // Prepare request - std::vector requestDataT(N * DUMMY_MODEL_INPUT_SIZE); - for (int i = 0; i < N; ++i) { - std::transform(requestData.begin(), - requestData.end(), - requestDataT.begin() + DUMMY_MODEL_INPUT_SIZE * i, - [i](int x) { return x + i; }); - } - request.Clear(); - for (int i = 0; i < N; i++) { - tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName + std::to_string(i)]; - proto.set_dtype(tensorflow::DataType::DT_FLOAT); - proto.mutable_tensor_content()->assign((char*)(requestDataT.data() + i * DUMMY_MODEL_INPUT_SIZE), - DUMMY_MODEL_INPUT_SIZE * sizeof(float)); - proto.mutable_tensor_shape()->add_dim()->set_size(1); - proto.mutable_tensor_shape()->add_dim()->set_size(10); - } - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::OK); - for (int i = 0; i < N; i++) { - ASSERT_EQ(response.outputs().count(customPipelineOutputName + std::to_string(i)), 1); - } - auto responseData = requestDataT; - std::transform(requestDataT.begin(), requestDataT.end(), requestDataT.begin(), [](float& v) { return v + 1.0; }); - - float* expected_output = requestDataT.data(); - for (int i = 0; i < N; i++) { - float* actual_output = (float*)response.outputs().at(customPipelineOutputName + std::to_string(i)).tensor_content().data(); - const int dataLengthToCheck = DUMMY_MODEL_OUTPUT_SIZE * sizeof(float); - const float* expected_output_address_to_check = expected_output + i * DUMMY_MODEL_OUTPUT_SIZE; - EXPECT_EQ(0, std::memcmp(actual_output, expected_output_address_to_check, dataLengthToCheck)) - << "Comparison on node:" << i << " output failed" << std::endl - << readableError(expected_output_address_to_check, actual_output, DUMMY_MODEL_OUTPUT_SIZE); - } -} - -class DLNodeFirst : public DLNode { - std::vector& order; - -public: - DLNodeFirst(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, - ModelManager& modelManager, std::vector& order, std::unordered_map nodeOutputNameAlias = {}, - std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}) : - DLNode(nodeName, modelName, modelVersion, modelManager, nodeOutputNameAlias, demultiplyCount, gatherFromNode), - order(order) {} - ovms::Status execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) override { - auto status = DLNode::execute(sessionId, notifyEndQueue); - order.push_back(1); - return status; - } -}; - -class DLNodeDeferred : public DLNode { - std::vector& order; - -public: - DLNodeDeferred(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, - ModelManager& modelManager, std::vector& order, std::unordered_map nodeOutputNameAlias = {}, - std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}) : - DLNode(nodeName, modelName, modelVersion, modelManager, nodeOutputNameAlias, demultiplyCount, gatherFromNode), - order(order) {} - ovms::Status execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) override { - auto status = DLNode::execute(sessionId, notifyEndQueue); - order.push_back(2); - return status; - } -}; - -class DLNodeNext : public DLNode { - std::vector& order; - -public: - DLNodeNext(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, - ModelManager& modelManager, std::vector& order, std::unordered_map nodeOutputNameAlias = {}, - std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}) : - DLNode(nodeName, modelName, modelVersion, modelManager, nodeOutputNameAlias, demultiplyCount, gatherFromNode), - order(order) {} - ovms::Status execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) override { - auto status = DLNode::execute(sessionId, notifyEndQueue); - order.push_back(3); - return status; - } -}; - -TEST_F(EnsembleFlowTest, OrderOfScheduling) { - ConstructorEnabledModelManager managerWithDummyModel; - config.setNireq(1); - managerWithDummyModel.reloadModelWithVersions(config); - - tensor_map_t inputsInfoTmp; - inputsInfoTmp[customPipelineInputName] = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - auto input_node = std::make_unique>(&request, inputsInfoTmp); - - tensor_map_t outputsInfoTmp; - outputsInfoTmp[customPipelineOutputName + "_1"] = std::make_shared(customPipelineOutputName + "_1", - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - outputsInfoTmp[customPipelineOutputName + "_2"] = std::make_shared(customPipelineOutputName + "_2", - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"}); - auto output_node = std::make_unique>(&response, outputsInfoTmp); - - // DL Nodes - std::vector order; - auto node_1 = std::make_unique("dummy_node_1", dummyModelName, requestedModelVersion, managerWithDummyModel, order); - auto node_2 = std::make_unique("dummy_node_2", dummyModelName, requestedModelVersion, managerWithDummyModel, order); - auto node_3 = std::make_unique("dummy_node_3", dummyModelName, requestedModelVersion, managerWithDummyModel, order); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *node_1, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - - pipeline.connect(*node_1, *node_3, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*input_node, *node_2, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - - pipeline.connect(*node_2, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName + "_1"}}); - pipeline.connect(*node_3, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName + "_2"}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(output_node)); - - pipeline.push(std::move(node_1)); - pipeline.push(std::move(node_2)); - pipeline.push(std::move(node_3)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - std::vector expectedOrder = { - 1, // try to schedule node_1 with success - 2, // try to schedule node_2, defer (with order ticket #1) - 3, // after node_1 ends, try to run next node (node_3), defer with order ticket #2 - 2, // also try to schedule previously deferred nodes, node_2 gets scheduled with success - 3}; // node_2 ends, try to schedule previously deferred node_3 with success - int expectedOrderIt = 0; - int lastValue = 0; - for (int orderElement : order) { - if (orderElement != lastValue) { - EXPECT_EQ(orderElement, expectedOrder[expectedOrderIt]); - expectedOrderIt++; - } - lastValue = orderElement; - } - // This fragment above is implemented that way because amount of scheduling retries may differ between different machines - // depending on the inference time of the dummy model - /* - -----O1-----O3---- - O---< >----O - -----O2----------- - */ -} - -TEST_F(EnsembleFlowTest, FailInDLNodeSetInputsMissingInput) { - // Most basic configuration, just process single dummy model request - - // input dummy(fail in setInputs) output - // O------->O------->O - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}, {"NON_EXISTING_INPUT", "REQUIRED_IN_THEORY_OUTPUT"}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - EXPECT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::INVALID_MISSING_INPUT); -} - -TEST_F(EnsembleFlowTest, FailInDLNodeExecuteInputsMissingInput) { - // Most basic configuration, just process single dummy model request - - // input dummy(fail in execute) output - // O------->O------->O - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *model_node, {{customPipelineInputName, std::string(DUMMY_MODEL_INPUT_NAME) + "_NON_EXISTING_INPUT_NAME_IN_MODEL"}}); - pipeline.connect(*model_node, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node)); - pipeline.push(std::move(output_node)); - - EXPECT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), ovms::StatusCode::INVALID_MISSING_INPUT); -} - -class DLNodeFailInFetch : public DLNode { -public: - DLNodeFailInFetch(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, ModelManager& modelManager) : - DLNode(nodeName, modelName, modelVersion, modelManager, {}) {} - ovms::Status fetchResults(NodeSession& nodeSession, SessionResults& sessionResults) override { - // no release is called as in dl_node.cpp when on error path - DLNode::fetchResults(nodeSession, sessionResults); - return StatusCode::UNKNOWN_ERROR; - } -}; - -TEST_F(EnsembleFlowTest, FailInDLNodeFetchResults) { - // Most basic configuration, just process single dummy model request - - // input dummy(fail in fetch) output - // O------->O------->O - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto failInFetchNode = std::make_unique("failInFetch_node", dummyModelName, requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *failInFetchNode, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*failInFetchNode, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(failInFetchNode)); - pipeline.push(std::move(output_node)); - - auto status = pipeline.execute(DEFAULT_TEST_CONTEXT); - EXPECT_EQ(status, ovms::StatusCode::UNKNOWN_ERROR) << status.string(); -} - -TEST_F(EnsembleFlowTest, FailInDLNodeFetchResultsStreamIdReleasedForDeferredNode) { - // input dummy(fail in fetch) output - // O------->O------->O - // input dummy output - // O------->O------->O - ConstructorEnabledModelManager managerWithDummyModel; - config.setNireq(1); - managerWithDummyModel.reloadModelWithVersions(config); - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto failInFetchNode = std::make_unique("failInFetch_node", dummyModelName, requestedModelVersion, managerWithDummyModel); - auto modelNode = std::make_unique("dummy_node", dummyModelName, requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - - pipeline.connect(*input_node, *failInFetchNode, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*input_node, *modelNode, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*failInFetchNode, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - pipeline.connect(*modelNode, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName + "_NOT_IMPORTANT"}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(failInFetchNode)); - pipeline.push(std::move(modelNode)); - pipeline.push(std::move(output_node)); - - auto status = pipeline.execute(DEFAULT_TEST_CONTEXT); - EXPECT_EQ(status, ovms::StatusCode::UNKNOWN_ERROR) << status.string(); -} - -TEST_F(EnsembleFlowTest, CorrectPipelineDefinitionNodesValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O dummy node (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // dummy node (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithDummyModel), StatusCode::OK); -} - -TEST_F(EnsembleFlowTest, PipelineWithStringModelConnectionUnsupported) { - ConstructorEnabledModelManager managerWithStringModel; - ovms::ModelConfig config = NATIVE_STRING_MODEL_CONFIG; - ASSERT_EQ(managerWithStringModel.reloadModelWithVersions(config), StatusCode::OK_RELOADED); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "string_node_1", "passthrough_string", std::nullopt, {{PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, PASSTHROUGH_STRING_MODEL_OUTPUT_NAME}}}, - {NodeKind::DL, "string_node_2", "passthrough_string", std::nullopt, {{PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, PASSTHROUGH_STRING_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O string node 1 (PASSTHROUGH_STRING_MODEL_INPUT_NAME) - connections["string_node_1"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, PASSTHROUGH_STRING_MODEL_INPUT_NAME}}}}; - // string node 1 (PASSTHROUGH_STRING_MODEL_OUTPUT_NAME) O--------->O string node 2 (PASSTHROUGH_STRING_MODEL_INPUT_NAME) - connections["string_node_2"] = { - {"string_node_1", {{PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, PASSTHROUGH_STRING_MODEL_INPUT_NAME}}}}; - // string node 2 (PASSTHROUGH_STRING_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"string_node_2", {{PASSTHROUGH_STRING_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithStringModel), StatusCode::NOT_IMPLEMENTED); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionNodesWithModelBatchingModeAutoValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - config.setBatchingMode(AUTO); - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O dummy node (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // dummy node (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithDummyModel), StatusCode::FORBIDDEN_MODEL_DYNAMIC_PARAMETER); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionNodesWithModelShapeModeAutoValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - config.parseShapeParameter("auto"); - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O dummy node (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // dummy node (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithDummyModel), StatusCode::FORBIDDEN_MODEL_DYNAMIC_PARAMETER); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionNodesWithMissingNodeModelValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node1", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::DL, "dummy_node2", "missing", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O dummy node 1 (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node1"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // request (customPipelineInputName) O--------->O dummy node 2 (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node2"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // dummy node (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"dummy_node1", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName + "_1"}}}, - {"dummy_node2", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName + "_2"}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithDummyModel), StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionNodesWithMissingConnectionNodeValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O dummy node (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // missingNode (customPipelineInputName) O--------->O dummy node (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node"] = { - {"missingNode", {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // dummy node (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithDummyModel), StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_NODE); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionNodesWithNodeOutputMissingValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O dummy node (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // dummy node (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{"MISSING", customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithDummyModel), StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_DATA_SOURCE); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionNodesWithNodeModelInputMissingValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O dummy node (DUMMY_MODEL_INPUT_NAME) - // /\--------| - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_OUTPUT_NAME}}}, - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, "MISSING"}}}}; - - // dummy node (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithDummyModel), StatusCode::PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionNodeNotAllInputsConnectedValidation) { - ConstructorEnabledModelManager manager; - ModelConfig sumModelConfig = SUM_MODEL_CONFIG; - manager.reloadModelWithVersions(sumModelConfig); - - PipelineFactory factory; - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "sum_node", "sum", std::nullopt, {{SUM_MODEL_OUTPUT_NAME, SUM_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // Missing connection for SUM_MODEL_INPUT_NAME_2 - connections["sum_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, SUM_MODEL_INPUT_NAME_1}}}}; - - connections[EXIT_NODE_NAME] = { - {"sum_node", {{SUM_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(manager), StatusCode::PIPELINE_NOT_ALL_INPUTS_CONNECTED); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionShapesNotMatchBetweenDLModelTensorsValidation) { - ConstructorEnabledModelManager manager; - ModelConfig dummy_1x10 = config; - ModelConfig dummy_1x20 = config; - dummy_1x10.setName("dummy_1x10"); - dummy_1x20.setName("dummy_1x20"); - dummy_1x10.setBatchSize(std::nullopt); - dummy_1x20.setBatchSize(std::nullopt); - ASSERT_EQ(dummy_1x10.parseShapeParameter("(1,10)"), StatusCode::OK); - ASSERT_EQ(dummy_1x20.parseShapeParameter("(1,20)"), StatusCode::OK); - - ASSERT_EQ(manager.reloadModelWithVersions(dummy_1x10), StatusCode::OK_RELOADED); - ASSERT_EQ(manager.reloadModelWithVersions(dummy_1x20), StatusCode::OK_RELOADED); - - PipelineFactory factory; - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node_1x10", "dummy_1x10", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::DL, "dummy_node_1x20", "dummy_1x20", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummy_node_1x10"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["dummy_node_1x20"] = { - {"dummy_node_1x10", {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}}}; - - connections[EXIT_NODE_NAME] = { - {"dummy_node_1x20", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(manager), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionPrecisionsNotMatchBetweenDLModelTensorsValidation) { - ConstructorEnabledModelManager manager; - ModelConfig dummy_fp32 = config; - ModelConfig dummy_fp64 = DUMMY_FP64_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(dummy_fp32), StatusCode::OK_RELOADED); - ASSERT_EQ(manager.reloadModelWithVersions(dummy_fp64), StatusCode::OK_RELOADED); - - PipelineFactory factory; - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node_fp32", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::DL, "dummy_node_fp64", "dummy_fp64", std::nullopt, {{DUMMY_FP64_MODEL_OUTPUT_NAME, DUMMY_FP64_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummy_node_fp32"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections["dummy_node_fp64"] = { - {"dummy_node_fp32", {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_FP64_MODEL_INPUT_NAME}}}}; - - connections[EXIT_NODE_NAME] = { - {"dummy_node_fp64", {{DUMMY_FP64_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(manager), StatusCode::INVALID_PRECISION); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionMultipleConnectionsToModelInputValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O dummy node (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}, - {customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // dummy node (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithDummyModel), StatusCode::PIPELINE_MODEL_INPUT_CONNECTED_TO_MULTIPLE_DATA_SOURCES); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionExitNodeIsDependencyErrorValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - connections["dummy_node"] = { - {EXIT_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - std::unique_ptr pipelineDefinition = std::make_unique("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition->validateNodes(managerWithDummyModel), StatusCode::PIPELINE_EXIT_USED_AS_NODE_DEPENDENCY); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionComplexGraphWithNoCycleValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME}, - {NodeKind::DL, "dummy_node", "output"}, - {NodeKind::DL, "dummy_node1", "output"}, - {NodeKind::DL, "dummy_node2", "output"}, - {NodeKind::DL, "dummy_node3", "output"}, - {NodeKind::DL, "dummy_node4", "output"}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request O--------->O dummy node - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{"output", "input"}}}}; - - // dummy node O--------->O dummy node 1 - connections["dummy_node1"] = { - {"dummy_node", {{"output", "input"}}}}; - - // dummy node 1 O--------->O dummy node 2 - connections["dummy_node2"] = { - {"dummy_node1", {{"output", "input"}}}}; - - // dummy node 2 O-------->\/ - // dummy node 4 O--------->O response - connections[EXIT_NODE_NAME] = { - {"dummy_node2", {{"output", "input"}}}, - {"dummy_node4", {{"output", "input"}}}}; - - // request O--------->O dummy node 3 - connections["dummy_node3"] = { - {ENTRY_NODE_NAME, {{"output", "input"}}}}; - - // dummy node 3 O-------->\/ - // dummy node 2 O--------->O dummy node 4 - connections["dummy_node4"] = { - {"dummy_node3", {{"output", "input"}}}, - {"dummy_node2", {{"output", "input"}}}}; - - // Create pipeline definition - PipelineDefinition pipelineDefinition("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition.validateForCycles(), StatusCode::OK); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionComplexGrapgWithCycleValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME}, - {NodeKind::DL, "dummy_node", "output"}, - {NodeKind::DL, "dummy_node1", "output"}, - {NodeKind::DL, "dummy_node2", "output"}, - {NodeKind::DL, "dummy_node3", "output"}, - {NodeKind::DL, "dummy_node4", "output"}, - {NodeKind::DL, "dummy_node5", "output"}, - {NodeKind::DL, "dummy_node6", "output"}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request O--------->O dummy node - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{"output", "input"}}}}; - - // dummy node O--------->O dummy node 1 - connections["dummy_node1"] = { - {"dummy_node", {{"output", "input"}}}}; - - // dummy node 1 O--------->O dummy node 2 - connections["dummy_node2"] = { - {"dummy_node1", {{"output", "input"}}}}; - - // dummy node 2 O-------->\/ - // dummy node 6 O--------->O dummy node 3 - connections["dummy_node3"] = { - {"dummy_node2", {{"output", "input"}}}, - {"dummy_node6", {{"output", "input"}}}}; - - // dummy node 3 O-------->\/ - // dummy node 6 O--------->O response - connections[EXIT_NODE_NAME] = { - {"dummy_node3", {{"output", "input"}}}, - {"dummy_node6", {{"output", "input"}}}}; - - // request O--------->O dummy node 4 - connections["dummy_node4"] = { - {ENTRY_NODE_NAME, {{"output", "input"}}}}; - - // dummy node 3 O-------->\/ - // dummy node 4 O--------->O dummy node 5 - connections["dummy_node5"] = { - {"dummy_node4", {{"output", "input"}}}, - {"dummy_node3", {{"output", "input"}}}}; - - // dummy node 5 O--------->O dummy node 6 - connections["dummy_node6"] = { - {"dummy_node5", {{"output", "input"}}}}; - - // Create pipeline definition - PipelineDefinition pipelineDefinition("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition.validateForCycles(), StatusCode::PIPELINE_CYCLE_FOUND); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionContainingCycleValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME}, - {NodeKind::DL, "dummy_node", "output"}, - {NodeKind::DL, "dummy_node1", "output"}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request O--------->O dummy node - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{"output", "input"}}}}; - - // response O--------->O dummy node - connections["dummy_node"] = { - {EXIT_NODE_NAME, {{"output", "input"}}}}; - - // dummy node 1 O--------->O response - connections[EXIT_NODE_NAME] = { - {"dummy_node1", {{"output", "input"}}}}; - - // dummy node O--------->O dummy node 1 - connections["dummy_node1"] = { - {"dummy_node", {{"output", "input"}}}}; - - // Create pipeline definition - PipelineDefinition pipelineDefinition("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition.validateForCycles(), StatusCode::PIPELINE_CYCLE_FOUND); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionContainingNodeConnectedToItselfValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME}, - {NodeKind::DL, "dummy_node", "output"}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request O--------->O dummy node ----| - // /\-----| - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{"output", "input"}}}, - {"dummy_node", {{"output", "input"}}}}; - - // dummy node 1 O--------->O response - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{"output", "input"}}}}; - - // Create pipeline definition - PipelineDefinition pipelineDefinition("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition.validateForCycles(), StatusCode::PIPELINE_CYCLE_FOUND); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionContainingTwoCyclesValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME}, - {NodeKind::DL, "dummy_node", "output"}, - {NodeKind::DL, "dummy_node1", "output"}, - {NodeKind::DL, "dummy_node2", "output"}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request O--------->O dummy node - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{"output", "input"}}}}; - - // dummy node O--------->O dummy node - connections["dummy_node"] = { - {EXIT_NODE_NAME, {{"output", "input"}}}}; - - // dummy node 1 O--------->O response - connections[EXIT_NODE_NAME] = { - {"dummy_node1", {{"output", "input"}}}}; - - // dummy node O---------------\/ - // dummy node 2 O--------->dummy node 1 - connections["dummy_node1"] = { - {"dummy_node", {{"output", "input"}}}, - {"dummy_node2", {{"output", "input"}}}}; - - // dummy node 1 O--------->O dummy node 2 - connections["dummy_node2"] = { - {"dummy_node1", {{"output", "input"}}}}; - - // Create pipeline definition - PipelineDefinition pipelineDefinition("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition.validateForCycles(), StatusCode::PIPELINE_CYCLE_FOUND); -} - -TEST_F(EnsembleFlowTest, PipelineDefinitionContainingUnconnectedNodeValidation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME}, - {NodeKind::DL, "dummy_node", "output"}, - {NodeKind::DL, "dummy_node1", "output"}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request O--------->O dummy node - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{"output", "input"}}}}; - - // dummy node O--------->O response - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{"output", "input"}}}}; - - // Create pipeline definition - PipelineDefinition pipelineDefinition("my_new_pipeline", info, connections); - ASSERT_EQ(pipelineDefinition.validateForCycles(), StatusCode::PIPELINE_CONTAINS_UNCONNECTED_NODES); -} - -TEST_F(EnsembleFlowTest, SimplePipelineFactoryCreation) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - PipelineFactory factory; - - // Nodes - // request dummy_node response - // O--------->O---------->O - // dummy - // default - // Models/Versions - const std::string pipelineName = "my_new_pipeline"; - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - pipeline_connections_t connections; - - // request (customPipelineInputName) O--------->O dummy node (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - - // dummy node (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (customPipelineOutputName) - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - - // Create pipeline definition - ASSERT_EQ(factory.createDefinition(pipelineName, info, connections, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::OK); - - std::unique_ptr pipeline; - - // Create pipeline out of created definition - ASSERT_EQ(factory.create(pipeline, pipelineName, &request, &response, managerWithDummyModel), StatusCode::OK); - - // Execute pipeline - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - const int dummySeriallyConnectedCount = 1; - checkDummyResponse(dummySeriallyConnectedCount); -} - -TEST_F(EnsembleFlowTest, ParallelPipelineFactoryUsage) { - // Prepare manager - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - PipelineFactory factory; - - // Nodes - // dummy_node_N - // .-------->O----------v - // request O--------->O---------->O response - // *-------->O----------^ - // dummy - // default - // Models/Versions - - const int PARALLEL_DUMMY_NODES = 3; - const int PARALLEL_SIMULATED_REQUEST_COUNT = 30; - - // Simulate reading from pipeline_config.json - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - for (int i = 0; i < PARALLEL_DUMMY_NODES; i++) { - info.emplace_back(std::move(NodeInfo( - NodeKind::DL, - "dummy_node_" + std::to_string(i), - "dummy", - std::nullopt, - {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}))); - } - - pipeline_connections_t connections; - - for (int i = 0; i < PARALLEL_DUMMY_NODES; i++) { - // request (customPipelineInputName) O--------->O dummy_node_N (DUMMY_MODEL_INPUT_NAME) - connections["dummy_node_" + std::to_string(i)] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - } - - // dummy_node_0 (DUMMY_MODEL_OUTPUT_NAME) O---------v - // dummy_node_1 (DUMMY_MODEL_OUTPUT_NAME) O--------->O response (output_0, output_1, output_N) - // dummy_node_N (DUMMY_MODEL_OUTPUT_NAME) O---------^ - auto& responseConnections = connections[EXIT_NODE_NAME]; - for (int i = 0; i < PARALLEL_DUMMY_NODES; i++) { - responseConnections["dummy_node_" + std::to_string(i)] = {{DUMMY_MODEL_OUTPUT_NAME, "output_" + std::to_string(i)}}; - } - - // Create pipeline definition - ASSERT_EQ(factory.createDefinition("my_new_pipeline", info, connections, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::OK); - - auto run = [&]() { - std::unique_ptr pipeline; - PredictResponse response_local; - - // Create pipeline out of created definition - ASSERT_EQ(factory.create(pipeline, "my_new_pipeline", &request, &response_local, managerWithDummyModel), StatusCode::OK); - - // Execute pipeline - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - - // Validate response - ASSERT_EQ(response_local.outputs_size(), PARALLEL_DUMMY_NODES); - - auto responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [](float& v) { v += 1.0; }); - - size_t expectedContentSize = DUMMY_MODEL_OUTPUT_SIZE * sizeof(float); - - for (int i = 0; i < PARALLEL_DUMMY_NODES; i++) { - std::string outputName = "output_" + std::to_string(i); - ASSERT_EQ(response_local.outputs().count(outputName), 1); - const auto& tensor = response_local.outputs().at(outputName); - ASSERT_EQ(tensor.tensor_content().size(), expectedContentSize); - float* actual_output = (float*)tensor.tensor_content().data(); - float* expected_output = responseData.data(); - - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, expectedContentSize)); - } - }; - - std::vector> promises(PARALLEL_SIMULATED_REQUEST_COUNT); - std::vector threads; - - for (int n = 0; n < PARALLEL_SIMULATED_REQUEST_COUNT; n++) { - threads.emplace_back(std::thread([&promises, n, &run]() { - promises[n].get_future().get(); - run(); - })); - } - - // Sleep to allow all threads to initialize - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - - for (auto& promise : promises) { - promise.set_value(); - } - - for (auto& thread : threads) { - thread.join(); - } -} - -TEST_F(EnsembleFlowTest, PipelineFactoryWrongConfiguration_MultipleEntryNodes) { - // Prepare manager - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - PipelineFactory factory; - - std::vector info{ - {NodeKind::ENTRY, "request1"}, - {NodeKind::ENTRY, "request2"}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - ASSERT_EQ(factory.createDefinition("pipeline", info, {}, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::PIPELINE_MULTIPLE_ENTRY_NODES); -} - -TEST_F(EnsembleFlowTest, PipelineFactoryWrongConfiguration_MultipleExitNodes) { - // Prepare manager - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - PipelineFactory factory; - - std::vector info{ - {NodeKind::EXIT, "response1"}, - {NodeKind::EXIT, "response2"}, - {NodeKind::ENTRY, ENTRY_NODE_NAME}, - }; - - ASSERT_EQ(factory.createDefinition("pipeline", info, {}, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::PIPELINE_MULTIPLE_EXIT_NODES); -} - -TEST_F(EnsembleFlowTest, PipelineFactoryWrongConfiguration_ExitMissing) { - // Prepare manager - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - PipelineFactory factory; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME}, - }; - - EXPECT_EQ(factory.createDefinition("pipeline", info, {}, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT); -} - -TEST_F(EnsembleFlowTest, PipelineFactoryWrongConfiguration_EntryMissing) { - // Prepare manager - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - PipelineFactory factory; - - std::vector info{ - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - EXPECT_EQ(factory.createDefinition("pipeline", info, {}, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT); -} - -TEST_F(EnsembleFlowTest, PipelineFactoryWrongConfiguration_DefinitionMissing) { - ConstructorEnabledModelManager manager; - PipelineFactory factory; - - PredictRequest request; - PredictResponse response; - std::unique_ptr pipeline; - EXPECT_EQ(factory.create(pipeline, "pipeline", &request, &response, manager), StatusCode::PIPELINE_DEFINITION_NAME_MISSING); -} - -TEST_F(EnsembleFlowTest, PipelineFactoryWrongConfiguration_NodeNameDuplicate) { - // Prepare manager - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - PipelineFactory factory; - - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME}, - {NodeKind::DL, "dummy_node", "dummy"}, - {NodeKind::DL, "dummy_node", "dummy"}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - - ASSERT_EQ(factory.createDefinition("pipeline", info, {}, managerWithDummyModel, managerWithDummyModel, managerWithDummyModel), StatusCode::PIPELINE_NODE_NAME_DUPLICATE); -} - -static const std::string PIPELINE_1_DUMMY_NAME = "pipeline1Dummy"; - -static const char* pipelineOneDummyConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithInputOutputsMappings) { - std::string fileToReload = directoryPath + "/ovms_config_file.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.loadConfig(fileToReload); - std::unique_ptr pipeline; - auto status = managerWithDummyModel.getPipelineFactory().create(pipeline, - "pipeline1Dummy", - &request, - &response, - managerWithDummyModel); - ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - const int dummySeriallyConnectedCount = 1; - checkDummyResponse(dummySeriallyConnectedCount); -} - -static const char* pipelineOneDummyConfig2ParallelDummy = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 2 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - }, - { - "name": "dummyNode2", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output2"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - }, - {"custom_dummy_output2": {"node_name": "dummyNode2", - "data_item": "new_dummy_output2"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithInputOutputsMappings2ParallelDummy) { - std::string fileToReload = directoryPath + "/ovms_config_file.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig2ParallelDummy), fileToReload); - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.loadConfig(fileToReload); - std::unique_ptr pipeline; - auto status = managerWithDummyModel.getPipelineFactory().create(pipeline, - "pipeline1Dummy", - &request, - &response, - managerWithDummyModel); - ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - ASSERT_EQ(response.outputs().count(customPipelineOutputName), 1); - ASSERT_EQ(response.outputs().count(std::string(customPipelineOutputName) + "2"), 1); - // check 1st output - const auto& output_proto = response.outputs().at(customPipelineOutputName); - const int batchSize = 1; - const int seriesLength = 1; - ASSERT_EQ(output_proto.tensor_content().size(), batchSize * DUMMY_MODEL_OUTPUT_SIZE * sizeof(float)); - ASSERT_EQ(output_proto.tensor_shape().dim_size(), 2); - ASSERT_EQ(output_proto.tensor_shape().dim(0).size(), batchSize); - ASSERT_EQ(output_proto.tensor_shape().dim(1).size(), DUMMY_MODEL_OUTPUT_SIZE); - - auto responseData = requestData; - std::for_each(responseData.begin(), responseData.end(), [seriesLength](float& v) { v += 1.0 * seriesLength; }); - - float* actual_output = (float*)output_proto.tensor_content().data(); - float* expected_output = responseData.data(); - const int dataLengthToCheck = DUMMY_MODEL_OUTPUT_SIZE * batchSize * sizeof(float); - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, dataLengthToCheck)) - << readableError(expected_output, actual_output, dataLengthToCheck); - - // check 2nd output - const auto& output_proto2 = response.outputs().at(customPipelineOutputName); - - ASSERT_EQ(output_proto2.tensor_content().size(), batchSize * DUMMY_MODEL_OUTPUT_SIZE * sizeof(float)); - ASSERT_EQ(output_proto2.tensor_shape().dim_size(), 2); - ASSERT_EQ(output_proto2.tensor_shape().dim(0).size(), batchSize); - ASSERT_EQ(output_proto2.tensor_shape().dim(1).size(), DUMMY_MODEL_OUTPUT_SIZE); - - actual_output = (float*)output_proto2.tensor_content().data(); - EXPECT_EQ(0, std::memcmp(actual_output, expected_output, dataLengthToCheck)) - << readableError(expected_output, actual_output, dataLengthToCheck); -} - -static const char* pipelineOneDummyConfigWrongNodeKind = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL modeloze Wrong kind", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithWrongNodeKind) { - performWrongPipelineConfigTest(pipelineOneDummyConfigWrongNodeKind); -} - -static const char* pipelineOneDummyConfigMissingNodeModelName = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}} - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithMissingNodeModelName) { - performWrongPipelineConfigTest(pipelineOneDummyConfigMissingNodeModelName); -} - -static const char* pipelineOneDummyConfigMissingNodeName = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}} - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithMissingNodeName) { - performWrongPipelineConfigTest(pipelineOneDummyConfigMissingNodeName); -} - -static const char* pipelineOneDummyConfigMissingNodeInputs = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}} - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithMissingNodeInputs) { - performWrongPipelineConfigTest(pipelineOneDummyConfigMissingNodeInputs); -} - -static const char* pipelineOneDummyConfigWithMissingNodeOutputs = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}} - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithMissingNodeOutputs) { - performWrongPipelineConfigTest(pipelineOneDummyConfigWithMissingNodeOutputs); -} - -static const char* pipelineOneDummyConfigWithMissingPipelineOutputs = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}} - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithMissingPipelineOutputs) { - performWrongPipelineConfigTest(pipelineOneDummyConfigWithMissingPipelineOutputs); -} - -static const char* pipelineOneDummyConfigWithMissingPipelineInputs = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}} - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithMissingPipelineInputs) { - performWrongPipelineConfigTest(pipelineOneDummyConfigWithMissingPipelineInputs); -} - -TEST_F(EnsembleFlowTest, ErrorHandlingSkipsDeferredNodesExecutionIfExecutionFailed) { - // This test creates specific scenario where 3 parallel nodes are getting executed - // with nireq=1. The second node gets stream id ticket for inference and is deferred - // for execution later. Meanwhile error occurs in third parallel node (shape validation error). - - // Expected result - have pipeline cancelled with proper error code - - // Manager with dummy model and nireq=1 - ConstructorEnabledModelManager managerWithDummyModel; - config.setNireq(1); - managerWithDummyModel.reloadModelWithVersions(config); - - // Configure pipeline - const tensor_map_t inputsInfo{{"proto_input_1x10", - std::make_shared("proto_input_1x10", - ovms::Precision::FP32, - DUMMY_MODEL_SHAPE_META, - Layout{"NC"})}, - {"proto_input_1x5", - std::make_shared("proto_input_1x5", - ovms::Precision::FP32, - ovms::Shape{1, 5}, - Layout{"NC"})}}; - auto input_node = std::make_unique>(&request, inputsInfo); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - - auto dummy_node_1 = std::make_unique("dummy_node_1", dummyModelName, requestedModelVersion, managerWithDummyModel); - auto dummy_node_2 = std::make_unique("dummy_node_2", dummyModelName, requestedModelVersion, managerWithDummyModel); - auto dummy_node_3 = std::make_unique("dummy_node_3", dummyModelName, requestedModelVersion, managerWithDummyModel); - - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *dummy_node_1, {{"proto_input_1x10", DUMMY_MODEL_INPUT_NAME}}); // this node will start execution, reserve stream id - pipeline.connect(*input_node, *dummy_node_2, {{"proto_input_1x10", DUMMY_MODEL_INPUT_NAME}}); // this node will start execution, get future object for stream id, defer to queue - pipeline.connect(*input_node, *dummy_node_3, {{"proto_input_1x5", DUMMY_MODEL_INPUT_NAME}}); // this node will fail at validation time - pipeline.connect(*dummy_node_1, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, "proto_output_1x10_A"}}); - pipeline.connect(*dummy_node_2, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, "proto_output_1x10_B"}}); - pipeline.connect(*dummy_node_3, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, "proto_output_1x5"}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(output_node)); - pipeline.push(std::move(dummy_node_1)); - pipeline.push(std::move(dummy_node_2)); - pipeline.push(std::move(dummy_node_3)); - - request.Clear(); - - auto& proto_input_1x5 = (*request.mutable_inputs())["proto_input_1x5"]; - auto& proto_input_1x10 = (*request.mutable_inputs())["proto_input_1x10"]; - - proto_input_1x5.set_dtype(tensorflow::DataType::DT_FLOAT); - proto_input_1x10.set_dtype(tensorflow::DataType::DT_FLOAT); - - std::vector data_1x5(5); - std::vector data_1x10(10); - std::iota(data_1x5.begin(), data_1x5.end(), 0); // 0, 1, 2, 3, 4 - std::iota(data_1x10.begin(), data_1x10.end(), 5); // 5, 6, ..., 14 - - proto_input_1x5.mutable_tensor_content()->assign((char*)data_1x5.data(), data_1x5.size() * sizeof(float)); - proto_input_1x5.mutable_tensor_shape()->add_dim()->set_size(1); - proto_input_1x5.mutable_tensor_shape()->add_dim()->set_size(data_1x5.size()); - - proto_input_1x10.mutable_tensor_content()->assign((char*)data_1x10.data(), data_1x10.size() * sizeof(float)); - proto_input_1x10.mutable_tensor_shape()->add_dim()->set_size(1); - proto_input_1x10.mutable_tensor_shape()->add_dim()->set_size(data_1x10.size()); - - EXPECT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); -} - -TEST_F(EnsembleFlowTest, ReloadPipelineDefinitionWithNewModelNameShouldPass) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - const std::string pipelineName = "originalName"; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - pipeline_connections_t connections; - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - PipelineDefinition pd(pipelineName, info, connections); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_TRUE(status.ok()); - - config.setName("newDummy"); - status = managerWithDummyModel.reloadModelWithVersions(config); - ASSERT_TRUE(status.ok()) << status.string(); - std::vector infoNew{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "newDummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - status = pd.reload(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel, std::move(infoNew), std::move(connections)); - EXPECT_TRUE(status.ok()) << status.string(); -} -const std::string notifierDetails{"UnusedNotifierDetails"}; - -TEST_F(EnsembleFlowTest, ReloadPipelineDefinitionWithNewNonExistingModelNameShouldFail) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - const std::string pipelineName = "originalName"; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - pipeline_connections_t connections; - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - PipelineDefinition pd(pipelineName, info, connections); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_TRUE(status.ok()); - - ASSERT_TRUE(status.ok()) << status.string(); - std::vector infoNew{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "newDummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - status = pd.reload(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel, std::move(infoNew), std::move(connections)); - EXPECT_EQ(status, ovms::StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL) << status.string(); -} - -TEST_F(EnsembleFlowTest, ReloadPipelineDefinitionWithAllModelVersionsRetiredShouldFail) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - const std::string pipelineName = "originalName"; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - pipeline_connections_t connections; - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - PipelineDefinition pd(pipelineName, info, connections); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_TRUE(status.ok()) << status.string(); - managerWithDummyModel.findModelByName("dummy")->retireAllVersions(); - - status = pd.reload(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel, std::move(info), std::move(connections)); - EXPECT_EQ(status, ovms::StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL) << status.string(); -} - -TEST_F(EnsembleFlowTest, RevalidatePipelineDefinitionWhen1ModelVersionBecomesAvailableShouldPass) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - const std::string pipelineName = "originalName"; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - pipeline_connections_t connections; - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - PipelineDefinition pd(pipelineName, info, connections); - pd.makeSubscriptions(managerWithDummyModel); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_TRUE(status.ok()) << status.string(); - managerWithDummyModel.findModelByName("dummy")->retireAllVersions(); - - status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_EQ(status, ovms::StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL) << status.string(); - - status = managerWithDummyModel.reloadModelWithVersions(config); - ASSERT_TRUE(status.ok()) << status.string(); - status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - EXPECT_TRUE(status.ok()) << status.string(); -} - -TEST_F(EnsembleFlowTest, RetirePipelineDefinitionExecuteShouldFail) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - const std::string pipelineName = "originalName"; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - pipeline_connections_t connections; - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - PipelineDefinition pd(pipelineName, info, connections); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_TRUE(status.ok()); - pd.retire(managerWithDummyModel); - std::unique_ptr pipeline; - status = pd.create(pipeline, &request, &response, managerWithDummyModel); - EXPECT_EQ(status, ovms::StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE); -} - -TEST_F(EnsembleFlowTest, ExecuteOnPipelineCreatedBeforeRetireShouldPass) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - const std::string pipelineName = "originalName"; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - pipeline_connections_t connections; - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - PipelineDefinition pd(pipelineName, info, connections); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_TRUE(status.ok()); - std::unique_ptr pipelineBeforeRetire; - status = pd.create(pipelineBeforeRetire, &request, &response, managerWithDummyModel); - ASSERT_TRUE(status.ok()); - pd.retire(managerWithDummyModel); - pipelineBeforeRetire->execute(DEFAULT_TEST_CONTEXT); - uint32_t dummySeriallyConnectedCount = 1; - checkDummyResponse(dummySeriallyConnectedCount); -} - -TEST_F(EnsembleFlowTest, RuntimeWrongBatchSizeArbitraryPosition) { - ConstructorEnabledModelManager managerWithDummyModel; - - ModelConfig configCN = DUMMY_MODEL_CONFIG; - configCN.setName("dummy_C1_N10"); - configCN.setBatchingParams(""); - configCN.parseShapeParameter("(1,10)"); - ASSERT_EQ(configCN.parseLayoutParameter("cn"), StatusCode::OK); - managerWithDummyModel.reloadModelWithVersions(configCN); - - configCN = DUMMY_MODEL_CONFIG; - configCN.setName("dummy_C1_N15"); - configCN.setBatchingParams(""); - configCN.parseShapeParameter("(1,15)"); - ASSERT_EQ(configCN.parseLayoutParameter("cn"), StatusCode::OK); - managerWithDummyModel.reloadModelWithVersions(configCN); - - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - Shape{1, 15}, - Layout{"CN"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - Shape{1, 10}, - Layout{"CN"}); - - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node_1 = std::make_unique("dummy_node_1", "dummy_C1_N10", requestedModelVersion, managerWithDummyModel); - auto model_node_2 = std::make_unique("dummy_node_2", "dummy_C1_N15", requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node_1, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_1, *model_node_2, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_2, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node_1)); - pipeline.push(std::move(model_node_2)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_BATCH_SIZE); -} - -TEST_F(EnsembleFlowTest, RuntimeWrongShapeArbitraryBatchPosition) { - ConstructorEnabledModelManager managerWithDummyModel; - - ModelConfig configCN = DUMMY_MODEL_CONFIG; - configCN.setName("dummy_C1_N10"); - configCN.setBatchingParams(""); - configCN.parseShapeParameter("(1,10)"); - ASSERT_EQ(configCN.parseLayoutParameter("cn"), StatusCode::OK); - managerWithDummyModel.reloadModelWithVersions(configCN); - - configCN = DUMMY_MODEL_CONFIG; - configCN.setName("dummy_C2_N10"); - configCN.setBatchingParams(""); - configCN.parseShapeParameter("(2,10)"); - ASSERT_EQ(configCN.parseLayoutParameter("cn"), StatusCode::OK); - managerWithDummyModel.reloadModelWithVersions(configCN); - - dagDummyModelOutputTensorInfo = std::make_shared(customPipelineOutputName, - ovms::Precision::FP32, - Shape{2, 10}, - Layout{"CN"}); - dagDummyModelInputTensorInfo = std::make_shared(customPipelineInputName, - ovms::Precision::FP32, - Shape{1, 10}, - Layout{"CN"}); - - // Configure pipeline - const tensor_map_t inputsInfo{{customPipelineInputName, dagDummyModelInputTensorInfo}}; - auto input_node = std::make_unique>(&request, inputsInfo); - auto model_node_1 = std::make_unique("dummy_node_1", "dummy_C1_N10", requestedModelVersion, managerWithDummyModel); - auto model_node_2 = std::make_unique("dummy_node_2", "dummy_C2_N10", requestedModelVersion, managerWithDummyModel); - const tensor_map_t outputsInfo{{customPipelineOutputName, dagDummyModelOutputTensorInfo}}; - auto output_node = std::make_unique>(&response, outputsInfo); - Pipeline pipeline(*input_node, *output_node, *this->reporter); - pipeline.connect(*input_node, *model_node_1, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_1, *model_node_2, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - pipeline.connect(*model_node_2, *output_node, {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}); - - pipeline.push(std::move(input_node)); - pipeline.push(std::move(model_node_1)); - pipeline.push(std::move(model_node_2)); - pipeline.push(std::move(output_node)); - - ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); -} - -class MockedPipelineDefinitionWithHandlingStatus : public PipelineDefinition { -public: - MockedPipelineDefinitionWithHandlingStatus(const std::string& pipelineName, - const std::vector& nodeInfos, - const pipeline_connections_t& connections) : - PipelineDefinition(pipelineName, nodeInfos, connections) {} - PipelineDefinitionStatus& getControlableStatus() { - return status; - } -}; - -TEST_F(EnsembleFlowTest, WaitForLoadingPipelineDefinitionFromBeginStatus) { - ConstructorEnabledModelManager managerWithDummyModel; - managerWithDummyModel.reloadModelWithVersions(config); - - const std::string pipelineName = "originalName"; - std::vector info{ - {NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {{customPipelineInputName, customPipelineInputName}}}, - {NodeKind::DL, "dummy_node", "dummy", std::nullopt, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_OUTPUT_NAME}}}, - {NodeKind::EXIT, EXIT_NODE_NAME}, - }; - std::unordered_map> connections; - connections["dummy_node"] = { - {ENTRY_NODE_NAME, {{customPipelineInputName, DUMMY_MODEL_INPUT_NAME}}}}; - connections[EXIT_NODE_NAME] = { - {"dummy_node", {{DUMMY_MODEL_OUTPUT_NAME, customPipelineOutputName}}}}; - MockedPipelineDefinitionWithHandlingStatus pd(pipelineName, info, connections); - pd.makeSubscriptions(managerWithDummyModel); - std::unique_ptr pipelineBeforeRetire; - std::thread t([&managerWithDummyModel, &pd]() { - std::this_thread::sleep_for(std::chrono::microseconds(PipelineDefinition::WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS / 4)); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_TRUE(status.ok()); - SPDLOG_INFO("Made pd validated"); - }); - auto status = pd.create(pipelineBeforeRetire, &request, &response, managerWithDummyModel); - ASSERT_TRUE(status.ok()) << status.string(); - pd.getControlableStatus().handle(UsedModelChangedEvent(notifierDetails)); - pd.getControlableStatus().handle(ValidationFailedEvent()); - status = pd.create(pipelineBeforeRetire, &request, &response, managerWithDummyModel); - ASSERT_EQ(status, ovms::StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET) << status.string(); - pd.getControlableStatus().handle(UsedModelChangedEvent(notifierDetails)); - status = pd.create(pipelineBeforeRetire, &request, &response, managerWithDummyModel); - ASSERT_EQ(status, ovms::StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET) << status.string(); - std::thread t2([&managerWithDummyModel, &pd]() { - std::this_thread::sleep_for(std::chrono::microseconds(PipelineDefinition::WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS / 4)); - auto status = pd.validate(managerWithDummyModel, managerWithDummyModel, managerWithDummyModel); - ASSERT_TRUE(status.ok()) << status.string(); - SPDLOG_INFO("Made pd validated"); - }); - status = pd.create(pipelineBeforeRetire, &request, &response, managerWithDummyModel); - ASSERT_TRUE(status.ok()) << status.string(); - uint32_t dummySeriallyConnectedCount = 1; - pipelineBeforeRetire->execute(DEFAULT_TEST_CONTEXT); - checkDummyResponse(dummySeriallyConnectedCount); - t.join(); - t2.join(); -} - -static const char* configJsonWithNoPipeline = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ] -})"; - -TEST_F(EnsembleFlowTest, RetireAllPipelinesAfterLoading) { - std::string fileToReload = directoryPath + "/ovms_config_file.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - createConfigFileWithContent(configJsonWithNoPipeline, fileToReload); - manager.loadConfig(fileToReload); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::RETIRED); -} -static const char* pipelineOneDummyConfigWithChangedInputName = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["NEW_INPUT_NAME"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "NEW_INPUT_NAME"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; -const std::string NEW_INPUT_NAME = "NEW_INPUT_NAME"; - -TEST_F(EnsembleFlowTest, ReloadPipelineAfterLoadingSuccessfullyChangedInputName) { - std::string fileToReload = directoryPath + "/ovms_config_file.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - - auto pdPtr = manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME); - auto inputsInfoBefore = pdPtr->getInputsInfo(); - ASSERT_EQ(inputsInfoBefore.count(NEW_INPUT_NAME), 0); - - // now reload - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfigWithChangedInputName), fileToReload); - manager.loadConfig(fileToReload); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - auto inputsInfoAfter = pdPtr->getInputsInfo(); - ASSERT_TRUE(status.ok()) << status.string(); - EXPECT_EQ(inputsInfoAfter.count(NEW_INPUT_NAME), 1); -} -static const char* pipelineOneDummyConfigWithMissingModel = R"( -{ - "model_config_list": [ - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; -TEST_F(EnsembleFlowTest, ReloadPipelineAfterLoadingFailDueToMissingModel) { - std::string fileToReload = directoryPath + "/ovms_config_file.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfigWithMissingModel), fileToReload); - manager.loadConfig(fileToReload); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED); -} -static const char* pipelineOneDummyConfigWithCorruptedModel = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy-wrong-path-to-model", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; -TEST_F(EnsembleFlowTest, ReloadPipelineAfterLoadingFailDueToCorruptedModel) { - std::string fileToReload = directoryPath + "/ovms_config_file.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfigWithCorruptedModel), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_EQ(status, StatusCode::PATH_INVALID); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED); - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - manager.loadConfig(fileToReload); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); -} -static const char* pipelineTwoDummyConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipelineToRetire", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - }, - { - "name": "pipelineToReload", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; -static const char* pipelineTwoDummyConfigAfterChanges = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipelineToAdd", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - }, - { - "name": "pipelineToReload", - "inputs": ["NEW_INPUT_NAME"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "NEW_INPUT_NAME"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -const std::string PIPELINE_TO_RETIRE{"pipelineToRetire"}; -const std::string PIPELINE_TO_RELOAD{"pipelineToReload"}; -const std::string PIPELINE_TO_ADD{"pipelineToAdd"}; - -TEST_F(EnsembleFlowTest, RetireReloadAddPipelineAtTheSameTime) { - // First add 2 pipelines with different names - // Then change config in a way: - // * remove 1 pipeline - // * change connection name between 2 nodes - // * add new pipeline (just with different name) - std::string fileToReload = directoryPath + "/ovms_config_file.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineTwoDummyConfig), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_TO_RETIRE)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_TO_RELOAD)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_TO_ADD), nullptr); - - auto pipelineToReloadPtr = manager.getPipelineFactory().findDefinitionByName(PIPELINE_TO_RELOAD); - auto inputsInfoBefore = pipelineToReloadPtr->getInputsInfo(); - ASSERT_EQ(inputsInfoBefore.count(NEW_INPUT_NAME), 0); - - // now reload - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineTwoDummyConfigAfterChanges), fileToReload); - status = manager.loadConfig(fileToReload); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_TO_RETIRE)->getStateCode(), - PipelineDefinitionStateCode::RETIRED); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_TO_RELOAD)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_TO_ADD)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - - auto inputsInfoAfter = pipelineToReloadPtr->getInputsInfo(); - EXPECT_EQ(inputsInfoAfter.count(NEW_INPUT_NAME), 1); -} - -static const char* pipelineOneDynamicParamDummyConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1, - "shape": "auto" - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, EnablingDynamicParametersForModelUsedInPipeline) { - /* - This test modifies config.json to enable dynamic parameters for model used in pipeline. - Test ensures such change will not invalidate pipeline. - Test ensures model have no dynamic parameters applied. - */ - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDynamicParamDummyConfig), fileToReload); - status = manager.loadConfig(fileToReload); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - - auto instance = manager.findModelInstance("dummy"); - ASSERT_NE(instance, nullptr); - ASSERT_FALSE(instance->getModelConfig().isDynamicParameterEnabled()); - ASSERT_EQ(instance->getStatus().getState(), ModelVersionState::AVAILABLE); -} - -static const char* dummyWithDynamicParamConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1, - "shape": "auto" - } - } - ] -})"; - -TEST_F(EnsembleFlowTest, EnablingDynamicParametersAndRemovingPipeline) { - /* - This test modifies config.json to enable dynamic parameters for model used in pipeline. - In the same time, we remove pipeline from config file. - Test ensures such change is valid and model will be reloaded and dynamic parameters will be applied. - Test ensures pipeline gets retired. - */ - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(dummyWithDynamicParamConfig), fileToReload); - status = manager.loadConfig(fileToReload); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::RETIRED); - - auto instance = manager.findModelInstance("dummy"); - ASSERT_NE(instance, nullptr); - ASSERT_TRUE(instance->getModelConfig().isDynamicParameterEnabled()); - ASSERT_EQ(instance->getStatus().getState(), ModelVersionState::AVAILABLE); -} - -static const char* pipelineModelSameNameConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - }, - { - "config": { - "name": "pipeline1Dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1, - "shape": "auto" - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -static const char* pipelineModelSameNameConfigNoPipeline = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - }, - { - "config": { - "name": "pipeline1Dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1, - "shape": "auto" - } - } - ] -})"; - -#if (MEDIAPIPE_DISABLE == 0) -static const char* mediapipeSameNameConfigMediapipe = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "mediapipe_config_list": [ - { - "name":"dummy", - "graph_path":"/ovms/src/test/mediapipe/graphdummy.pbtxt" - } - ] -})"; -static const char* mediapipeSameNameConfigMediapipeWithPipeline = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummyModel", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "mediapipe_config_list": [ - { - "name":"dummy", - "graph_path":"/ovms/src/test/mediapipe/graphdummy.pbtxt" - } - ], - "pipeline_config_list": [ - { - "name": "dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummyModel", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; -static const std::string MEDIAPIPE_DUMMY_NAME = "dummy"; -TEST_F(EnsembleFlowTest, MediapipeConfigModelWithSameName) { - // Expected result - model added, adding pipeline failed - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(mediapipeSameNameConfigMediapipe), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_EQ(status, StatusCode::MEDIAPIPE_GRAPH_NAME_OCCUPIED); - ASSERT_FALSE(manager.getMediapipeFactory().definitionExists(MEDIAPIPE_DUMMY_NAME)); - - auto instance = manager.findModelInstance(MEDIAPIPE_DUMMY_NAME); - ASSERT_NE(instance, nullptr); - ASSERT_EQ(instance->getStatus().getState(), ModelVersionState::AVAILABLE); -} - -TEST_F(EnsembleFlowTest, MediapipeConfigModelWithSameNamePipeline) { - // Expected result - model added, adding pipeline failed - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(mediapipeSameNameConfigMediapipeWithPipeline), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_EQ(status, StatusCode::MEDIAPIPE_GRAPH_NAME_OCCUPIED); - - ASSERT_FALSE(manager.getMediapipeFactory().definitionExists(MEDIAPIPE_DUMMY_NAME)); - - ASSERT_TRUE(manager.servableExists(MEDIAPIPE_DUMMY_NAME, ServableQueryType::Pipeline)); -} -#endif -TEST_F(EnsembleFlowTest, PipelineConfigModelWithSameName) { - // Expected result - model added, adding pipeline failed - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineModelSameNameConfig), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_EQ(status, StatusCode::PIPELINE_NAME_OCCUPIED); - - ASSERT_FALSE(manager.getPipelineFactory().definitionExists(PIPELINE_1_DUMMY_NAME)); - - auto instance = manager.findModelInstance(PIPELINE_1_DUMMY_NAME); - ASSERT_NE(instance, nullptr); - ASSERT_EQ(instance->getStatus().getState(), ModelVersionState::AVAILABLE); -} - -TEST_F(EnsembleFlowTest, ModelLoadedAddPipelineWithSameName) { - // Expected result - adding pipeline failed - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineModelSameNameConfigNoPipeline), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - - auto instance = manager.findModelInstance(PIPELINE_1_DUMMY_NAME); - ASSERT_NE(instance, nullptr); - ASSERT_EQ(instance->getStatus().getState(), ModelVersionState::AVAILABLE); - - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineModelSameNameConfig), fileToReload); - status = manager.loadConfig(fileToReload); - - ASSERT_FALSE(manager.getPipelineFactory().definitionExists(PIPELINE_1_DUMMY_NAME)); - - instance = manager.findModelInstance(PIPELINE_1_DUMMY_NAME); - ASSERT_NE(instance, nullptr); - ASSERT_EQ(instance->getStatus().getState(), ModelVersionState::AVAILABLE); -} - -TEST_F(EnsembleFlowTest, PipelineLoadedAddModelWithSameName) { - // Expected result - adding model failed - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineModelSameNameConfig), fileToReload); - status = manager.loadConfig(fileToReload); - - ASSERT_TRUE(manager.getPipelineFactory().definitionExists(PIPELINE_1_DUMMY_NAME)); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - - auto instance = manager.findModelInstance(PIPELINE_1_DUMMY_NAME); - ASSERT_EQ(instance, nullptr); -} - -TEST_F(EnsembleFlowTest, PipelineRetiredAddModelWithSameName) { - // Expected result - adding model failed - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ConstructorEnabledModelManager manager; - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineModelSameNameConfigNoPipeline), fileToReload); - status = manager.loadConfig(fileToReload); - - ASSERT_TRUE(manager.getPipelineFactory().definitionExists(PIPELINE_1_DUMMY_NAME)); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::RETIRED); - - auto instance = manager.findModelInstance(PIPELINE_1_DUMMY_NAME); - ASSERT_EQ(instance, nullptr); -} - -static const char* pipelinePipelineSameNameConfig = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - }, - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode2", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineAddSecondPipelineWithSameName) { - // Expected result - adding second pipeline fails - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineOneDummyConfig), fileToReload); - ConstructorEnabledModelManager manager; - - auto status = manager.loadConfig(fileToReload); - ASSERT_TRUE(status.ok()) << status.string(); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelinePipelineSameNameConfig), fileToReload); - status = manager.loadConfig(fileToReload); - - ASSERT_TRUE(manager.getPipelineFactory().definitionExists(PIPELINE_1_DUMMY_NAME)); - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); - auto& nodeInfos = manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getNodeInfos(); - ASSERT_FALSE(std::find_if(nodeInfos.begin(), nodeInfos.end(), [](auto nodeInfo) { return nodeInfo.nodeName == "dummyNode"; }) == nodeInfos.end()); - ASSERT_TRUE(std::find_if(nodeInfos.begin(), nodeInfos.end(), [](auto nodeInfo) { return nodeInfo.nodeName == "dummyNode2"; }) == nodeInfos.end()); -} - -static const char* pipelineDemultiplexerShapeNotEqualToDemultiplyCount = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(3, 2, 10) ", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ], - "demultiply_count": 2 - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TYPED_TEST(EnsembleFlowBothApiTest, DemultiplexerMultipleBatchSizeWithShapeNotEqualToDemultiplyCountNotAllowed) { - std::string fileToReload = this->directoryPath + "/config.json"; - std::string ovmsConfig = std::string(pipelineDemultiplexerShapeNotEqualToDemultiplyCount); - adjustConfigForTargetPlatform(ovmsConfig); - - createConfigFileWithContent(ovmsConfig, fileToReload); - ConstructorEnabledModelManager manager; - - auto status = manager.loadConfig(fileToReload); - ASSERT_EQ(status, StatusCode::PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED); -} - -static const char* pipelineInnerNodeConnectionShapeRangeNotMatch = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy_A", - "base_path": "/ovms/src/test/dummy", - "shape": "(-1,30:40) ", - "nireq": 1 - } - }, - { - "config": { - "name": "dummy_B", - "base_path": "/ovms/src/test/dummy", - "shape": "(-1,41:60) ", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode_A", - "model_name": "dummy_A", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - }, - { - "name": "dummyNode_B", - "model_name": "dummy_B", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "dummyNode_A", - "data_item": "new_dummy_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode_B", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, InnerNodeConnectionShapeRangeNotMatch) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineInnerNodeConnectionShapeRangeNotMatch), fileToReload); - ConstructorEnabledModelManager manager; - - auto status = manager.loadConfig(fileToReload); - ASSERT_EQ(status, StatusCode::INVALID_SHAPE) << status.string(); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED); -} - -static const char* pipelineInnerNodeConnectionShapeRangePartiallyMatch = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy_A", - "base_path": "/ovms/src/test/dummy", - "shape": "(-1,30:40) ", - "nireq": 1 - } - }, - { - "config": { - "name": "dummy_B", - "base_path": "/ovms/src/test/dummy", - "shape": "(-1,40:60) ", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode_A", - "model_name": "dummy_A", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - }, - { - "name": "dummyNode_B", - "model_name": "dummy_B", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "dummyNode_A", - "data_item": "new_dummy_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode_B", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, InnerNodeConnectionShapeRangePartiallyMatch) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineInnerNodeConnectionShapeRangePartiallyMatch), fileToReload); - ConstructorEnabledModelManager manager; - - auto status = manager.loadConfig(fileToReload); - ASSERT_EQ(status, StatusCode::OK); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); -} - -static const char* pipelineDemultiplexerShapeEqualToDemultiplyCount = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(2, 2, 10) ", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "new_dummy_output"} - ], - "demultiply_count": 2 - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "new_dummy_output"} - } - ] - } - ] -})"; - -TYPED_TEST(EnsembleFlowBothApiTest, DemultiplexerMultipleBatchSizeWithShapeEqualToDemultiplyCountAllowed) { - std::string fileToReload = this->directoryPath + "/config.json"; - std::string ovmsConfig = std::string(pipelineDemultiplexerShapeEqualToDemultiplyCount); - adjustConfigForTargetPlatform(ovmsConfig); - - createConfigFileWithContent(ovmsConfig, fileToReload); - ConstructorEnabledModelManager manager; - - auto status = manager.loadConfig(fileToReload); - ASSERT_EQ(status, StatusCode::OK); - - ASSERT_EQ(manager.getPipelineFactory().findDefinitionByName(PIPELINE_1_DUMMY_NAME)->getStateCode(), - PipelineDefinitionStateCode::AVAILABLE); -} - -static const char* pipelineSingleIncrement4DimInputNHWC = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1,2,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, ExecuteSingleIncrement4DimInputNHWC) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimInputNHWC), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareRequest({1.0, 2.0, 3.0, 4.0, 5.0, 6.0}, request, "pipeline_input", {1, 1, 2, 3}); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {2.0, 5.0, 3.0, 6.0, 4.0, 7.0}, response, {1, 3, 1, 2}); -} - -static const char* pipelineSingleIncrement4DimInputNHWCDynamicBatch = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1,2,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "demultiply_count": 0, - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TYPED_TEST(EnsembleFlowBothApiTest, ExecuteSingleIncrement4DimInputNHWCDynamicBatch) { - std::string fileToReload = this->directoryPath + "/config.json"; - std::string ovmsConfig = std::string(pipelineSingleIncrement4DimInputNHWCDynamicBatch); - adjustConfigForTargetPlatform(ovmsConfig); - - createConfigFileWithContent(ovmsConfig, fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - this->prepareRequest({1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0}, - this->request, "pipeline_input", {2, 1, 1, 2, 3}); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &this->request, &this->response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse(std::string{"pipeline_output"}, {2.0, 5.0, 3.0, 6.0, 4.0, 7.0, 11.0, 41.0, 21.0, 51.0, 31.0, 61.0}, this->response, {2, 1, 3, 1, 2}); -} - -static const char* pipelineSingleIncrement4DimOutputNHWC = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,3,1,2) ", - "layout": {"output": "nhwc:nchw"}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, ExecuteSingleIncrement4DimOutputNHWC) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareRequest({1.0, 2.0, 3.0, 4.0, 5.0, 6.0}, request, "pipeline_input", {1, 3, 1, 2}); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {2.0, 4.0, 6.0, 3.0, 5.0, 7.0}, response, {1, 1, 2, 3}); -} - -static const char* pipelineSingleIncrement4DimOutputNHWCDynamicBatch = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,3,1,2) ", - "layout": {"output": "nhwc:nchw"}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "demultiply_count": 0, - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TYPED_TEST(EnsembleFlowBothApiTest, ExecuteSingleIncrement4DimOutputNHWCDynamicBatch) { - std::string fileToReload = this->directoryPath + "/config.json"; - std::string ovmsConfig = std::string(pipelineSingleIncrement4DimOutputNHWCDynamicBatch); - adjustConfigForTargetPlatform(ovmsConfig); - - createConfigFileWithContent(ovmsConfig, fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - this->prepareRequest({1.0, 2.0, 3.0, 4.0, 5.0, 6.0, - 10.0, 20.0, 30.0, 40.0, 50.0, 60.0}, - this->request, "pipeline_input", {2, 1, 3, 1, 2}); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &this->request, &this->response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {2.0, 4.0, 6.0, 3.0, 5.0, 7.0, 11.0, 31.0, 51, 21.0, 41.0, 61.0}, this->response, {2, 1, 1, 2, 3}); -} - -static const char* pipelineAmbiguousInputMeta = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment_nhwc", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1,2,3) ", - "layout": {"input": "nhwc:nchw"}, - "nireq": 1 - } - }, - { - "config": { - "name": "increment_nchw", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,3,1,2) ", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node_nhwc", - "model_name": "increment_nhwc", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - }, - { - "name": "increment_node_nchw", - "model_name": "increment_nchw", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output_nhwc": {"node_name": "increment_node_nhwc", - "data_item": "out"} - }, - {"pipeline_output_nchw": {"node_name": "increment_node_nchw", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineAmbiguousInputMetaFailsToLoad) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineAmbiguousInputMeta), fileToReload); - ConstructorEnabledModelManager manager; - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::PIPELINE_INPUTS_AMBIGUOUS_METADATA); -} - -static const char* pipelineInnerConnectedNhwc = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment_nchw_in_nhwc_out", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,3,1,2) ", - "layout": {"output": "nhwc:nchw"}, - "nireq": 1 - } - }, - { - "config": { - "name": "increment_nhwc_in_nchw_out", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1,2,3) ", - "layout": {"input": "nhwc:nchw"}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node_nchw_in_nhwc_out", - "model_name": "increment_nchw_in_nhwc_out", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - }, - { - "name": "increment_node_nhwc_in_nchw_out", - "model_name": "increment_nhwc_in_nchw_out", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "increment_node_nchw_in_nhwc_out", - "data_item": "out"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node_nhwc_in_nchw_out", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, ExecutePipelineWithInnerNhwcConnection) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineInnerConnectedNhwc), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareRequest({1.0, 2.0, 3.0, 4.0, 5.0, 6.0}, request, "pipeline_input", {1, 3, 1, 2}); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {3.0, 4.0, 5.0, 6.0, 7.0, 8.0}, response, {1, 3, 1, 2}); -} - -class EnsembleFlowTestBinaryInput : public EnsembleFlowTest { -public: - const std::string imagePath = getGenericFullPathForSrcTest("/ovms/src/test/binaryutils/rgb.jpg"); - const std::string imagePath2x2 = getGenericFullPathForSrcTest("/ovms/src/test/binaryutils/rgb2x2.jpg"); - const std::string imagePath4x4 = getGenericFullPathForSrcTest("/ovms/src/test/binaryutils/rgb4x4.jpg"); - const std::string graycaleImagePath = getGenericFullPathForSrcTest("/ovms/src/test/binaryutils/grayscale.jpg"); -}; - -static const char* pipelineSingleIncrement4DimOutputNHWC1x1 = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1,1,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, BatchSize1) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC1x1), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareBinaryRequest(imagePath, request, "pipeline_input"); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {37.0, 28.0, 238.0}, response, {1, 3, 1, 1}); -} - -static const char* pipelineWith4DimDummyFP64 = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/dummy_fp64", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1,1,3) ", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input:0": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output:0", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, DoublePrecision) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWith4DimDummyFP64), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareBinaryRequest(imagePath, request, "pipeline_input"); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {37.0, 28.0, 238.0}, response, {1, 1, 1, 3}); -} - -static const char* pipelineSingleIncrement4DimOutputNHWC1x1BatchAny = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(-1,1,1,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, BatchSizeAny) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC1x1BatchAny), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - const size_t batchSize = 100; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimShape("pipeline_output", response, {batchSize, 3, 1, 1}); -} - -static const char* pipelineSingleIncrement4DimOutputNCHW1x1 = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,3,1,1) ", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, NchwEntryNotSupported) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNCHW1x1), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareBinaryRequest(imagePath, request, "pipeline_input"); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - auto status = pipeline->execute(DEFAULT_TEST_CONTEXT); - ASSERT_EQ(status, StatusCode::INVALID_NO_OF_CHANNELS) << status.string(); -} - -static const char* pipelineSingleIncrement4DimOutputNHWC1x1Grayscale = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1,1,1) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, GrayscaleImage) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC1x1Grayscale), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareBinaryRequest(graycaleImagePath, request, "pipeline_input"); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {1.0}, response, {1, 1, 1, 1}); -} - -static const char* pipelineSingleIncrement4DimOutputNHWC1x1BS5 = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(5,1,1,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, BatchSize5) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC1x1BS5), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 5; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0}, response, {5, 3, 1, 1}); -} - -static const char* pipelineSingleIncrement4DimOutputNHWC2x2 = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,2,2,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, ResizeBatch1) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC2x2), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareBinaryRequest(imagePath, request, "pipeline_input"); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {37.0, 37.0, 37.0, 37.0, 28.0, 28.0, 28.0, 28.0, 238.0, 238.0, 238.0, 238.0}, response, {1, 3, 2, 2}); -} - -static const char* pipelineSingleIncrement4DimOutputNHWC2x2BS5 = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(5,2,2,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, ResizeBatch5) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC2x2BS5), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 5; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {37.0, 37.0, 37.0, 37.0, 28.0, 28.0, 28.0, 28.0, 238.0, 238.0, 238.0, 238.0, 37.0, 37.0, 37.0, 37.0, 28.0, 28.0, 28.0, 28.0, 238.0, 238.0, 238.0, 238.0, 37.0, 37.0, 37.0, 37.0, 28.0, 28.0, 28.0, 28.0, 238.0, 238.0, 238.0, 238.0, 37.0, 37.0, 37.0, 37.0, 28.0, 28.0, 28.0, 28.0, 238.0, 238.0, 238.0, 238.0, 37.0, 37.0, 37.0, 37.0, 28.0, 28.0, 28.0, 28.0, 238.0, 238.0, 238.0, 238.0}, response, {5, 3, 2, 2}); -} - -static const char* pipelineSingleIncrement4DimOutputNHWC1Channel = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1,1,1) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, ColorChannelsDiffer) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC1Channel), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareBinaryRequest(imagePath, request, "pipeline_input"); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_NO_OF_CHANNELS); -} - -TEST_F(EnsembleFlowTestBinaryInput, InvalidData) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC1x1), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - request.Clear(); - tensorflow::TensorProto& inputProto = (*request.mutable_inputs())["pipeline_input"]; - inputProto.set_dtype(tensorflow::DataType::DT_STRING); - inputProto.add_string_val("INVALID_IMAGE"); - inputProto.mutable_tensor_shape()->add_dim()->set_size(1); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::IMAGE_PARSING_FAILED); -} - -static const char* pipelineSingleIncrement4DimOutputNHWC1x1EntryDemultiplexer = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1,1,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "demultiply_count": 0, - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, EntryDemultiplexer) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWC1x1EntryDemultiplexer), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 5; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0}, response, {5, 1, 3, 1, 1}); -} - -static const char* pipelineSingleIncrement4DimOutputNHWCRangeResolutionEntryStaticDemultiplexer = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1:3,1:3,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "demultiply_count": 5, - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, EntryStaticDemultiplexerResolutionMatches) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWCRangeResolutionEntryStaticDemultiplexer), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 5; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0}, response, {5, 1, 3, 1, 1}); -} - -TEST_F(EnsembleFlowTestBinaryInput, EntryStaticDemultiplexerResolutionAutoAlign) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWCRangeResolutionEntryStaticDemultiplexer), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 5; - prepareBinaryRequest(imagePath4x4, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimShape("pipeline_output", response, {5, 1, 3, 3, 3}); -} - -static const char* pipelineSingleIncrement4DimOutputNHWCRangeResolutionEntryDynamicDemultiplexer = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1:3,1:3,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "increment_pipeline", - "inputs": ["pipeline_input"], - "demultiply_count": 0, - "nodes": [ - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, EntryDynamicDemultiplexerResolutionMatches) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWCRangeResolutionEntryDynamicDemultiplexer), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 5; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0, 37.0, 28.0, 238.0}, response, {5, 1, 3, 1, 1}); -} - -TEST_F(EnsembleFlowTestBinaryInput, EntryDynamicDemultiplexerResolutionResolutionMismatch) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleIncrement4DimOutputNHWCRangeResolutionEntryDynamicDemultiplexer), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 5; - prepareBinaryRequest(imagePath4x4, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "increment_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); -} - -static const char* pipelineWithOnlyDynamicCustomNode = R"( -{ - "model_config_list": [], - "custom_node_library_config_list": [ - { - "name": "lib_dynamic_image", - "base_path": "/ovms/bazel-bin/src/lib_node_dynamic_image.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_dynamic_image", - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -// This test ensure binary inputs work for pipelines with layout ANY. -// Such pipelines have only custom nodes as entry nodes. -// In this case we do not reject the request but create NHWC content out of that. -TEST_F(EnsembleFlowTestBinaryInput, BinaryInputWithPipelineInputLayoutANY_RequestBS1) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWithOnlyDynamicCustomNode), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 1; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {44.0, 35.0, 245.0}, response, {1, 1, 1, 3}); -} - -TEST_F(EnsembleFlowTestBinaryInput, BinaryInputWithPipelineInputLayoutANY_RequestBS2) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWithOnlyDynamicCustomNode), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 2; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {44.0, 35.0, 245.0, 44.0, 35.0, 245.0}, response, {2, 1, 1, 3}); -} - -TEST_F(EnsembleFlowTestBinaryInput, BinaryInputWithPipelineInputLayoutANY_RequestMisaligned) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWithOnlyDynamicCustomNode), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareMisalignedBinaryImageRequest(imagePath, imagePath2x2, request, "pipeline_input"); - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH); -} - -TEST_F(EnsembleFlowTest, TensorContentInputWithPipelineInputLayoutANY_RequestNhwc) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWithOnlyDynamicCustomNode), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareRequest({1.0, 2.0, 3.0, 4.0}, request, "pipeline_input", {1, 4, 1}); // should be [1, 4, 1, 1] - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS); -} - -static const char* pipelineWithOnlyDynamicCustomNodeAndDemultiplexer = R"( -{ - "model_config_list": [], - "custom_node_library_config_list": [ - { - "name": "lib_dynamic_image", - "base_path": "/ovms/bazel-bin/src/lib_node_dynamic_image.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "demultiply_count": 0, - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_dynamic_image", - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "custom_node", - "data_item": "custom_node_output"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, BinaryInputWithPipelineInputLayoutANYAndDemultiplexer_RequestBS1) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWithOnlyDynamicCustomNodeAndDemultiplexer), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 1; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {44.0, 35.0, 245.0}, response, {1, 1, 1, 1, 3}); -} - -TEST_F(EnsembleFlowTestBinaryInput, BinaryInputWithPipelineInputLayoutANYAndDemultiplexer_RequestBS2) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWithOnlyDynamicCustomNodeAndDemultiplexer), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 2; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {44.0, 35.0, 245.0, 44.0, 35.0, 245.0}, response, {2, 1, 1, 1, 3}); -} - -TEST_F(EnsembleFlowTestBinaryInput, BinaryInputWithPipelineInputLayoutANYAndDemultiplexer_RequestMisaligned) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWithOnlyDynamicCustomNodeAndDemultiplexer), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - prepareMisalignedBinaryImageRequest(imagePath2x2, imagePath, request, "pipeline_input"); - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH); -} - -TYPED_TEST(EnsembleFlowBothApiTest, TensorContentInputWithPipelineInputLayoutANYAndDemultiplexer_RequestNhwc) { - std::string fileToReload = this->directoryPath + "/config.json"; - std::string ovmsConfig = std::string(pipelineWithOnlyDynamicCustomNodeAndDemultiplexer); - adjustConfigForTargetPlatform(ovmsConfig); - - createConfigFileWithContent(ovmsConfig, fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - this->prepareRequest({1.0, 2.0, 3.0, 4.0}, this->request, "pipeline_input", {1, 1, 4, 1}); // should be [1, 1, 4, 1, 1] - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &this->request, &this->response, manager), StatusCode::OK); - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS); -} - -static const char* pipelineWithDynamicCustomNodeDemultiplexerAndDynamicResolutionModel = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,-1,-1,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "custom_node_library_config_list": [ - { - "name": "lib_dynamic_image", - "base_path": "/ovms/bazel-bin/src/lib_node_dynamic_image.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "demultiply_count": 0, - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_dynamic_image", - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - }, - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, BinaryInputWithPipelineInputLayoutANYCustomNodeDemultiplexerAndDynamicResolutionModel) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWithDynamicCustomNodeDemultiplexerAndDynamicResolutionModel), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - int batchSize = 1; - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {45.0, 36.0, 246.0}, response, {1, 1, 3, 1, 1}); -} - -static const char* pipelineWithDynamicCustomNodeDemultiplexerAndRangeOfResolutionModel = R"( -{ - "model_config_list": [ - { - "config": { - "name": "increment", - "base_path": "/ovms/src/test/increment_1x3x4x5", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "shape": "(1,1:2,1:2,3) ", - "layout": "nhwc:nchw", - "nireq": 1 - } - } - ], - "custom_node_library_config_list": [ - { - "name": "lib_dynamic_image", - "base_path": "/ovms/bazel-bin/src/lib_node_dynamic_image.so" - } - ], - "pipeline_config_list": [ - { - "name": "my_pipeline", - "inputs": ["pipeline_input"], - "demultiply_count": 0, - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_dynamic_image", - "type": "custom", - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - }, - { - "name": "increment_node", - "model_name": "increment", - "type": "DL model", - "inputs": [ - {"input": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "output", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "increment_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTestBinaryInput, BinaryInputWithPipelineInputLayoutANYCustomNodeDemultiplexerAndRangeOfResolutionModel) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineWithDynamicCustomNodeDemultiplexerAndRangeOfResolutionModel), fileToReload); - ConstructorEnabledModelManager manager; - std::unique_ptr pipeline; - - // Try with resolution out of shape range, expect INVALID_SHAPE - int batchSize = 1; - prepareBinaryRequest(imagePath4x4, request, "pipeline_input", batchSize); - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::OK); - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::INVALID_SHAPE); - - request.Clear(); - response.Clear(); - - // Try with resolution matching the shape, expect OK - ASSERT_EQ(manager.getPipelineFactory().create(pipeline, "my_pipeline", &request, &response, manager), StatusCode::OK); - prepareBinaryRequest(imagePath, request, "pipeline_input", batchSize); - - ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - checkIncrement4DimResponse("pipeline_output", {45.0, 36.0, 246.0}, response, {1, 1, 3, 1, 1}); -} - -// Demultiplexer at request level (before model inference) -static const char* pipelineSingleStringModelWithDemultiplexerRequest = R"( -{ - "model_config_list": [ - { - "config": { - "name": "passthrough_string", - "base_path": "/ovms/src/test/passthrough_string", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipe", - "inputs": ["pipeline_input"], - "demultiply_count": 0, - "nodes": [ - { - "name": "pipe_node", - "model_name": "passthrough_string", - "type": "DL model", - "inputs": [ - {"my_name": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "my_name", - "alias": "out"} - ] - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "pipe_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineWithStringDemultiplexerRequestUnsupported) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleStringModelWithDemultiplexerRequest), fileToReload); - ConstructorEnabledModelManager manager; - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::PIPELINE_STRING_DEMUILTIPLICATION_UNSUPPORTED); -} - -// Demultiplexer at node level (after model inference) -static const char* pipelineSingleStringModelWithDemultiplexerNode = R"( -{ - "model_config_list": [ - { - "config": { - "name": "passthrough_string", - "base_path": "/ovms/src/test/passthrough_string", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - } - ], - "pipeline_config_list": [ - { - "name": "pipe", - "inputs": ["pipeline_input"], - "nodes": [ - { - "name": "pipe_node", - "model_name": "passthrough_string", - "type": "DL model", - "inputs": [ - {"my_name": {"node_name": "request", - "data_item": "pipeline_input"}} - ], - "outputs": [ - {"data_item": "my_name", - "alias": "out"} - ], - "demultiply_count": 0 - } - ], - "outputs": [ - {"pipeline_output": {"node_name": "pipe_node", - "data_item": "out"} - } - ] - } - ] -})"; - -TEST_F(EnsembleFlowTest, PipelineWithStringDemultiplexerNodeUnsupported) { - std::string fileToReload = directoryPath + "/config.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(pipelineSingleStringModelWithDemultiplexerNode), fileToReload); - ConstructorEnabledModelManager manager; - - ASSERT_EQ(manager.loadConfig(fileToReload), StatusCode::PIPELINE_STRING_DEMUILTIPLICATION_UNSUPPORTED); -} diff --git a/src/test/gather_node_test.cpp b/src/test/gather_node_test.cpp deleted file mode 100644 index cc0b87d87b..0000000000 --- a/src/test/gather_node_test.cpp +++ /dev/null @@ -1,376 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "../dags/dl_node.hpp" -#include "../dags/dlnodesession.hpp" -#include "../dags/entry_node.hpp" -#include "../dags/exit_node.hpp" -#include "../dags/gatherexitnodeinputhandler.hpp" -#include "../dags/gathernodeinputhandler.hpp" -#include "../capi_frontend/capi_dag_utils.hpp" -#include "../kfs_frontend/kfs_utils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" -#include "../dags/nodeinputhandler.hpp" -#include "../dags/nodestreamidguard.hpp" -#include "../dags/pipeline.hpp" -#include "../dags/pipeline_factory.hpp" -#include "../dags/pipelinedefinition.hpp" -#include "../logging.hpp" -#include "../modelconfig.hpp" -#include "../modelinstanceunloadguard.hpp" -#include "../modelinstance.hpp" -#include "../ov_utils.hpp" -#include "../prediction_service_utils.hpp" -#include "src/status.hpp" - -#include "constructor_enabled_model_manager.hpp" -#include "platform_utils.hpp" -#include "light_test_utils.hpp" -#include "test_utils.hpp" -#include "test_with_temp_dir.hpp" - -using namespace ovms; - -using testing::ElementsAre; -using testing::Return; - -class GatherNodeInputHandlerTest : public ::testing::Test {}; - -TEST_F(GatherNodeInputHandlerTest, ThreePredecessorNodesWithSubsessionSize2) { - // simulate all 3 inputs coming from different predecessor nodes - // with session demultiplexed to 2 shards - const uint32_t shardsCount = 2; // subsessionSize/demultiplyCount - std::vector inputNames{"a", "b"}; - std::vector> shapes{{1, 10}, {1, 2}}; - std::vector precisions{ov::element::Type_t::f32, ov::element::Type_t::f32}; - std::vector> tensorsData{{-1, 4, 5, 12, 3, 52, 12, 0.5, 9, 1.67}, {1., 3}}; - std::vector inputTensors{ - TensorWithSource(createTensorWithNoDataOwnership(precisions[0], shapes[0], tensorsData[0].data())), - TensorWithSource(createTensorWithNoDataOwnership(precisions[1], shapes[1], tensorsData[1].data()))}; - NodeSessionMetadata meta{DEFAULT_TEST_CONTEXT}; - const std::string demultiplexerName = "NOT_IMPORTANT_NAME"; - auto newMeta = meta.generateSubsessions(demultiplexerName, shardsCount)[0]; - auto [_, collapsingDetails] = newMeta.getCollapsedSessionMetadata({demultiplexerName}); - GatherNodeInputHandler gInputHandler(inputNames.size(), collapsingDetails); - for (session_id_t j = 0; j < shardsCount; ++j) { - for (size_t i = 0; i < inputNames.size(); ++i) { - EXPECT_FALSE(gInputHandler.isReady()); - gInputHandler.setInput(inputNames[i], inputTensors[i], j); - // each input coming from different node so we call notify each time - ASSERT_EQ(gInputHandler.notifyFinishedDependency(), StatusCode::OK); - } - } - EXPECT_TRUE(gInputHandler.isReady()); - const auto tensorMap = gInputHandler.getInputs(); - EXPECT_EQ(tensorMap.size(), inputNames.size()); - - std::vector> resultTensorsData(inputNames.size()); - for (size_t i = 0; i < inputNames.size(); ++i) { - resultTensorsData[i].reserve(tensorsData[i].size() * shardsCount); - std::copy(tensorsData[i].begin(), tensorsData[i].end(), resultTensorsData[i].begin()); - std::copy(tensorsData[i].begin(), tensorsData[i].end(), resultTensorsData[i].begin() + tensorsData[i].size()); - } - for (size_t i = 0; i < inputNames.size(); ++i) { - const auto& tensor = tensorMap.at(inputNames[i]); - EXPECT_EQ(tensor.get_size(), tensorsData[i].size() * shardsCount); - EXPECT_THAT(tensor.get_shape(), ElementsAre(shardsCount, 1, tensorsData[i].size())); - EXPECT_EQ(std::memcmp((char*)((const void*)(tensor.data())), resultTensorsData[i].data(), resultTensorsData[i].size() * sizeof(float)), 0); - } -} - -TEST_F(GatherNodeInputHandlerTest, GatheringOnTwoDemultiplexersAtOnce) { - const std::string inputName{"a"}; - const size_t elementCountPerShard = 10; - std::vector shape{1, elementCountPerShard}; - ov::element::Type_t precision{ov::element::Type_t::f32}; - const std::vector demultiplyCounts{3, 5}; // 3 for first demultiply, 5 for second - const std::vector demultiplexerNodeNames{"firstDemultiplexer", "secondDemultiplexer"}; - NodeSessionMetadata meta{DEFAULT_TEST_CONTEXT}; - auto firstLevelMetas = meta.generateSubsessions(demultiplexerNodeNames[0], demultiplyCounts[0]); - std::vector> metadatas(demultiplyCounts[0]); - for (size_t i = 0; i < demultiplyCounts[0]; ++i) { - metadatas[i] = firstLevelMetas[i].generateSubsessions(demultiplexerNodeNames[1], demultiplyCounts[1]); - } - - const size_t numberOfShards = std::accumulate(demultiplyCounts.begin(), demultiplyCounts.end(), 1, std::multiplies()); - const size_t numberOfElementsInGatheredTensor = elementCountPerShard * numberOfShards; - std::vector tensorsData(numberOfElementsInGatheredTensor); - std::iota(tensorsData.begin(), tensorsData.end(), 0.1); - std::vector inputTensors(numberOfShards); - GatherNodeInputHandler gInputHandler(1, {demultiplexerNodeNames, demultiplyCounts}); - Status status; - for (size_t i = 0; i < demultiplyCounts[0]; ++i) { - for (size_t j = 0; j < demultiplyCounts[1]; ++j) { - auto index = i * demultiplyCounts[1] + j; - auto tensor = TensorWithSource(createTensorWithNoDataOwnership(precision, shape, (void*)(tensorsData.data() + index * elementCountPerShard))); - ASSERT_FALSE(gInputHandler.isReady()); - SPDLOG_DEBUG("i: {}, j: {}, metadatas.size: {}, metadatas[i].size() :{}", i, j, metadatas.size(), metadatas[i].size()); - auto shardId = metadatas[i][j].getShardId({demultiplexerNodeNames[0], demultiplexerNodeNames[1]}); - status = gInputHandler.setInput(inputName, - tensor, - shardId); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - gInputHandler.notifyFinishedDependency(); - } - } - ASSERT_TRUE(gInputHandler.isReady()); - const auto tensorMap = gInputHandler.getInputs(); - ASSERT_EQ(tensorMap.size(), 1); - const auto& tensor = tensorMap.at(inputName); - EXPECT_EQ(tensor.get_size(), tensorsData.size()); - EXPECT_THAT(tensor.get_shape(), ElementsAre(demultiplyCounts[0], demultiplyCounts[1], 1, elementCountPerShard)); - EXPECT_EQ(std::memcmp((char*)((const void*)(tensor.data())), tensorsData.data(), tensorsData.size() * sizeof(float)), 0); -} - -TEST_F(GatherNodeInputHandlerTest, SetInputsWithShardsHavingDifferentShapesShouldReturnErrorWhenGathering) { - const std::string inputNames{"a"}; - std::vector> shapes{{1, 10}, {1, 9}}; - ov::element::Type_t precision{ov::element::Type_t::f32}; - std::vector tensorsData{-1, 4, 5, 12, 3, 52, 12, 0.5, 9, 1.67}; - std::vector inputTensors{ - TensorWithSource(createTensorWithNoDataOwnership(precision, shapes[0], tensorsData.data())), - TensorWithSource(createTensorWithNoDataOwnership(precision, shapes[1], tensorsData.data()))}; - const session_id_t shardsCount = 2; // subsessionSize/demultiplyCount - CollapseDetails collapsingDetails{{std::string("NOT_IMPORTANT_DEMULTIPLEXER_NAME")}, {shardsCount}}; - GatherNodeInputHandler gInputHandler(inputNames.size(), collapsingDetails); - Status status; - for (session_id_t j = 0; j < shardsCount; ++j) { - EXPECT_FALSE(gInputHandler.isReady()); - status = gInputHandler.setInput(inputNames, inputTensors[j], j); - EXPECT_EQ(status, StatusCode::OK) << status.string(); - // each input coming from different node so we call notify each time - status = gInputHandler.notifyFinishedDependency(); - if (!status.ok()) { - EXPECT_EQ(status, StatusCode::PIPELINE_INCONSISTENT_SHARD_DIMENSIONS) << status.string(); - break; - } - } - // The second notify should fail since the shard dimension should be different - EXPECT_EQ(status, StatusCode::PIPELINE_INCONSISTENT_SHARD_DIMENSIONS) << status.string(); -} - -class GatherNodeTest : public TestWithTempDir {}; - -static const char* configDummy1BsDummy2Bs = R"( -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1 - } - }, - { - "config": { - "name": "dummy2bs", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 1, - "batch_size": 2 - } - } - ] -})"; - -class DLNodeSessionWithGetInputsExposed : public DLNodeSession { -public: - DLNodeSessionWithGetInputsExposed(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelInstanceProvider& manager, const std::string& modelName, model_version_t modelVersion) : - DLNodeSession(metadata, nodeName, inputsCount, collapsingDetails, manager, modelName, modelVersion) {} - DLNodeSessionWithGetInputsExposed(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelInstanceProvider& manager, const std::string& modelName, model_version_t modelVersion) : - DLNodeSession(std::move(metadata), nodeName, inputsCount, collapsingDetails, manager, modelName, modelVersion) {} - - const auto& getInputs() const { - return this->inputHandler->getInputs(); - } -}; - -class DLNodeWithGetInputsExposed : public DLNode { -public: - DLNodeWithGetInputsExposed(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, - ModelManager& modelManager, - std::unordered_map nodeOutputNameAlias, const std::optional>& gatherFrom) : - DLNode(nodeName, modelName, modelVersion, modelManager, nodeOutputNameAlias, 0, gatherFrom.value_or(std::set())) { - } - const auto& getInputsFromInputHandler(session_key_t sessionId) const { - DLNodeSessionWithGetInputsExposed& dlnodesessionWithGetInputsExposed = static_cast(*nodeSessions.at(sessionId)); - return dlnodesessionWithGetInputsExposed.getInputs(); - } - std::unique_ptr createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) override { - return std::make_unique(metadata, getName(), previous.size(), collapsingDetails, - this->modelManager, this->modelName, this->modelVersion.value_or(0)); - } -}; - -TEST_F(GatherNodeTest, FullFlowGatherInNonExitNode) { - // This test simulates node with multiple subsessions connected to following node - // that should gather it results but is not exit node - ConstructorEnabledModelManager manager; - const std::string fileToReload = directoryPath + "/ovms_config_file.json"; - createConfigFileWithContent(adjustConfigForTargetPlatformCStr(configDummy1BsDummy2Bs), fileToReload); - auto status = manager.loadConfig(fileToReload); - ASSERT_EQ(status, StatusCode::OK) << status.string(); - const std::string node1Name = "node1"; - DLNode oneDummyNode1{node1Name, "dummy", 1, manager, {}}; - const std::string demultiplexerNodeName{"nodeDummy"}; - const std::optional> gatherFrom{{demultiplexerNodeName}}; - DLNodeWithGetInputsExposed gather2DummyNode{"nodeGather", "dummy2bs", 1, manager, {}, gatherFrom}; - Pipeline::connect(oneDummyNode1, gather2DummyNode, {{DUMMY_MODEL_OUTPUT_NAME, DUMMY_MODEL_INPUT_NAME}}); - - // prepare tensors to be gathered - const std::vector shape{1, 10}; - const ov::element::Type_t precision{ov::element::Type_t::f32}; - std::vector nodeRawResults1{-1, 4, 5, 12, 3, 52, 12, 0.5, 9, 1.67}; - std::vector nodeRawResults2{-13, -4.4, 15, 2, 0.3, -42, 13, 0.1, 91, 21.67}; - auto originalTensor1 = createTensorWithNoDataOwnership(precision, shape, nodeRawResults1.data()); - auto originalTensor2 = createTensorWithNoDataOwnership(precision, shape, nodeRawResults2.data()); - // prepare session results - TensorWithSourceMap dummy1Result{{DUMMY_MODEL_OUTPUT_NAME, TensorWithSource(originalTensor1)}}; - TensorWithSourceMap dummy2Result{{DUMMY_MODEL_OUTPUT_NAME, TensorWithSource(originalTensor2)}}; - NodeSessionMetadata meta{DEFAULT_TEST_CONTEXT}; - const session_id_t shardsCount = 2; - auto subsessions = meta.generateSubsessions(demultiplexerNodeName, shardsCount); - ASSERT_EQ(subsessions.size(), 2); - SessionResults oneDummyNodeSessionResults1; - SessionResults oneDummyNodeSessionResults2; - oneDummyNodeSessionResults1.insert({subsessions[0].getSessionKey(), {subsessions[0], dummy1Result}}); - oneDummyNodeSessionResults2.insert({subsessions[1].getSessionKey(), {subsessions[1], dummy2Result}}); - // actual test steps - ASSERT_EQ(gather2DummyNode.setInputs(oneDummyNode1, oneDummyNodeSessionResults1), StatusCode::OK); - ASSERT_EQ(gather2DummyNode.setInputs(oneDummyNode1, oneDummyNodeSessionResults2), StatusCode::OK); - auto readySessions = gather2DummyNode.getReadySessions(); - ASSERT_EQ(readySessions.size(), 1); - const auto& inputs = gather2DummyNode.getInputsFromInputHandler(subsessions[0].getSessionKey({demultiplexerNodeName})); - EXPECT_EQ(inputs.size(), 1); - ASSERT_NE(inputs.find(DUMMY_MODEL_INPUT_NAME), inputs.end()); - const auto& gatheredTensor = inputs.at(DUMMY_MODEL_INPUT_NAME); - EXPECT_EQ(gatheredTensor.get_size(), nodeRawResults1.size() * shardsCount); - std::vector resultTensorData(nodeRawResults1.size() * shardsCount); - std::copy(nodeRawResults1.begin(), nodeRawResults1.end(), resultTensorData.begin()); - std::copy(nodeRawResults2.begin(), nodeRawResults2.end(), resultTensorData.begin() + nodeRawResults1.size()); - EXPECT_EQ(memcmp((char*)((const void*)gatheredTensor.data()), resultTensorData.data(), resultTensorData.size() * sizeof(float)), 0); -} - -class GatherExitNodeInputHandlerTest : public ::testing::Test { -protected: - char* buffer = nullptr; - const std::string tensorName = "example_tensor_name"; - size_t requestedBufferSize = 20; -}; - -class TFSGatherExitNodeInputHandlerTest : public GatherExitNodeInputHandlerTest { -protected: - tensorflow::serving::PredictResponse response; -}; - -TEST_F(TFSGatherExitNodeInputHandlerTest, IsBufferSet) { - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::OK); - EXPECT_NE(buffer, nullptr); -} - -TEST_F(TFSGatherExitNodeInputHandlerTest, BufferPointsToDataInProto) { - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::OK); - auto it = response.mutable_outputs()->find(tensorName); - ASSERT_NE(it, response.mutable_outputs()->end()); - auto& proto = it->second; - EXPECT_EQ(proto.mutable_tensor_content()->data(), buffer); -} - -TEST_F(TFSGatherExitNodeInputHandlerTest, BufferHasCorrectSize) { - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::OK); - auto it = response.mutable_outputs()->find(tensorName); - ASSERT_NE(it, response.mutable_outputs()->end()); - auto& proto = it->second; - EXPECT_EQ(proto.mutable_tensor_content()->size(), requestedBufferSize); -} - -TEST_F(TFSGatherExitNodeInputHandlerTest, TensorAlreadyExistsInProto) { - auto& existingProto = (*response.mutable_outputs())[tensorName]; - (void)existingProto; - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::INTERNAL_ERROR); -} - -class KFSGatherExitNodeInputHandlerTest : public GatherExitNodeInputHandlerTest { -protected: - ::KFSResponse response; - - KFSTensorOutputProto* getPreparedTensor() { - KFSTensorOutputProto* ptr = nullptr; - for (int i = 0; i < response.outputs_size(); i++) { - auto* output = response.mutable_outputs(i); - if (output->name() == tensorName) { - ptr = output; - break; - } - } - return ptr; - } -}; - -TEST_F(KFSGatherExitNodeInputHandlerTest, IsBufferSet) { - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::OK); - EXPECT_NE(buffer, nullptr); -} - -TEST_F(KFSGatherExitNodeInputHandlerTest, HasTensorWithExpectedName) { - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::OK); - bool hasExpectedTensor = false; - for (int i = 0; i < response.outputs_size(); i++) { - auto* output = response.mutable_outputs(i); - if (output->name() == tensorName) { - hasExpectedTensor = true; - break; - } - } - ASSERT_TRUE(hasExpectedTensor); -} - -TEST_F(KFSGatherExitNodeInputHandlerTest, HasOneTensor) { - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::OK); - ASSERT_EQ(response.outputs_size(), 1); - ASSERT_EQ(response.raw_output_contents_size(), 1); -} - -TEST_F(KFSGatherExitNodeInputHandlerTest, ReturnedBufferMatchesRawOutputContentPtr) { - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::OK); - ASSERT_EQ(response.raw_output_contents_size(), 1); - ASSERT_EQ(response.mutable_raw_output_contents(0)->data(), buffer); -} - -TEST_F(KFSGatherExitNodeInputHandlerTest, BufferHasCorrectSizeBufferHasCorrectSize) { - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::OK); - ASSERT_EQ(response.raw_output_contents_size(), 1); - ASSERT_EQ(response.mutable_raw_output_contents(0)->size(), requestedBufferSize); -} - -TEST_F(KFSGatherExitNodeInputHandlerTest, TensorAlreadyExistsInProto) { - auto* existingProto = response.add_outputs(); - existingProto->set_name(tensorName); - response.add_raw_output_contents(); - ASSERT_EQ(prepareConsolidatedTensorImpl(&response, tensorName, ov::element::Type_t::i32, {1, 10}, buffer, requestedBufferSize), StatusCode::INTERNAL_ERROR); -} diff --git a/src/test/mediapipe/config_mp_tf_passthrough.json b/src/test/mediapipe/config_mp_tf_passthrough.json deleted file mode 100644 index 96ee7961a3..0000000000 --- a/src/test/mediapipe/config_mp_tf_passthrough.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "shape": "(-1, -1)" - } - }, - { - "config": { - "name": "scalar", - "base_path": "/ovms/src/test/scalar" - } - } - ], - "mediapipe_config_list": [ - { - "name": "mpTfsPassthrough", - "graph_path": "/ovms/src/test/mediapipe/graphtfpassthrough.pbtxt" - }, - { - "name": "mpTFDummy", - "graph_path": "/ovms/src/test/mediapipe/graphdummy_tf.pbtxt" - }, - { - "name": "mpTFScalar", - "graph_path": "/ovms/src/test/mediapipe/graphscalar_tf.pbtxt" - } - ] -} \ No newline at end of file diff --git a/src/test/mediapipe/graphdummy_tf.pbtxt b/src/test/mediapipe/graphdummy_tf.pbtxt deleted file mode 100644 index 23c3e3b429..0000000000 --- a/src/test/mediapipe/graphdummy_tf.pbtxt +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -input_stream: "TFTENSOR_1:in" -output_stream: "TFTENSOR:out" -node { - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "dummy" - servable_version: "1" - } - } -} -node { - calculator: "OpenVINOInferenceCalculator" - input_side_packet: "SESSION:session" - input_stream: "TFTENSOR_B:in" - output_stream: "TFTENSORA:out" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOInferenceCalculatorOptions]: { - tag_to_input_tensor_names { - key: "TFTENSOR_B" - value: "b" - } - tag_to_output_tensor_names { - key: "TFTENSORA" - value: "a" - } - } - } -} diff --git a/src/test/mediapipe/graphdummy_tflite.pbtxt b/src/test/mediapipe/graphdummy_tflite.pbtxt deleted file mode 100644 index 6c68838424..0000000000 --- a/src/test/mediapipe/graphdummy_tflite.pbtxt +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -input_stream: "TFLITE_TENSOR_1:in" -output_stream: "TFLITE_TENSOR:out" -node { - calculator: "OpenVINOModelServerSessionCalculator" - output_side_packet: "SESSION:session" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: { - servable_name: "dummy" - servable_version: "1" - } - } -} -node { - calculator: "OpenVINOInferenceCalculator" - input_side_packet: "SESSION:session" - input_stream: "TFLITE_TENSOR_B:in" - output_stream: "TFLITE_TENSORA:out" - node_options: { - [type.googleapis.com / mediapipe.OpenVINOInferenceCalculatorOptions]: { - tag_to_input_tensor_names { - key: "TFLITE_TENSOR_B" - value: "b" - } - tag_to_output_tensor_names { - key: "TFLITE_TENSORA" - value: "a" - } - } - } -} diff --git a/src/test/mediapipe/graphtfpassthrough.pbtxt b/src/test/mediapipe/graphtfpassthrough.pbtxt deleted file mode 100644 index bd571adbb7..0000000000 --- a/src/test/mediapipe/graphtfpassthrough.pbtxt +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright 2023 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -input_stream: "TFTENSOR:in" -output_stream: "TFTENSOR:out" -node { - calculator: "PassThroughCalculator" - input_stream: "TFTENSOR:in" - output_stream: "TFTENSOR:out" -} diff --git a/src/test/mediapipe/relative_paths/config_tflite_passthrough.json b/src/test/mediapipe/relative_paths/config_tflite_passthrough.json deleted file mode 100644 index 05d2276bb5..0000000000 --- a/src/test/mediapipe/relative_paths/config_tflite_passthrough.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "model_config_list": [], - "mediapipe_config_list": [ - { - "name":"mpTfLiteTensorDummy", - "graph_path":"/ovms/src/test/mediapipe/graphdummy_tflite.pbtxt", - "base_path":"/ovms/src/test/mediapipe/relative_paths/only_subconfig", - "subconfig":"subconfig.json" - } - ] -} diff --git a/src/test/mediapipe_framework_test.cpp b/src/test/mediapipe_framework_test.cpp index a085f02c27..a92a81d58c 100644 --- a/src/test/mediapipe_framework_test.cpp +++ b/src/test/mediapipe_framework_test.cpp @@ -45,7 +45,6 @@ #include "../server.hpp" #include "../shape.hpp" #include "../stringutils.hpp" -#include "src/tensorflow_type_utils.hpp" #include "src/timer.hpp" #include "constructor_enabled_model_manager.hpp" #include "c_api_test_utils.hpp" diff --git a/src/test/mediapipeflow_test.cpp b/src/test/mediapipeflow_test.cpp index 1d38181576..e0d7f6d069 100644 --- a/src/test/mediapipeflow_test.cpp +++ b/src/test/mediapipeflow_test.cpp @@ -36,7 +36,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "mediapipe/calculators/ovms/modelapiovmsadapter.hpp" +#include "src/mediapipe_calculators/ovms/modelapiovmsadapter.hpp" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/canonical_errors.h" #pragma GCC diagnostic pop @@ -64,7 +64,6 @@ #include "../shape.hpp" #include "../stringutils.hpp" #include "src/systeminfo.hpp" -#include "src/tensorflow_type_utils.hpp" #include "constructor_enabled_model_manager.hpp" #include "c_api_test_utils.hpp" #include "mediapipe/framework/formats/image_frame.h" @@ -294,13 +293,6 @@ class MediapipeFlowKfsTest : public MediapipeFlowTest { } }; -class MediapipeTFTest : public MediapipeFlowTest { -public: - void SetUp() { - SetUpServer("/ovms/src/test/mediapipe/config_mp_tf_passthrough.json"); - } -}; - class MediapipeTensorTest : public MediapipeFlowTest { public: void SetUp() { @@ -323,13 +315,6 @@ class MediapipeOvTensorPyTensorConverterTest : public MediapipeFlowTest { }; #endif -class MediapipeTfLiteTensorTest : public MediapipeFlowTest { -public: - void SetUp() { - SetUpServer("/ovms/src/test/mediapipe/relative_paths/config_tflite_passthrough.json"); - } -}; - class MediapipeEmbeddingsTest : public MediapipeFlowTest { public: void SetUp() { @@ -462,42 +447,6 @@ TEST_F(MediapipeOvTensorPyTensorConverterTest, Infer) { } #endif -TEST_F(MediapipeTFTest, Passthrough) { - const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); - KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); - ::KFSRequest request; - ::KFSResponse response; - - const std::string modelName{"mpTfsPassthrough"}; - request.Clear(); - response.Clear(); - inputs_info_t inputsMeta{{"in", {DUMMY_MODEL_SHAPE, precision}}}; - std::vector requestData{13.5, 0., 0, 0., 0., 0., 0., 0, 3., 67.}; - preparePredictRequest(request, inputsMeta, requestData); - request.mutable_model_name()->assign(modelName); - ASSERT_EQ(impl.ModelInfer(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - size_t dummysInTheGraph = 0; - checkDummyResponse("out", requestData, request, response, dummysInTheGraph, 1, modelName); -} - -TEST_F(MediapipeTFTest, DummyInfer) { - const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); - KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); - ::KFSRequest request; - ::KFSResponse response; - - const std::string modelName{"mpTFDummy"}; - request.Clear(); - response.Clear(); - inputs_info_t inputsMeta{{"in", {DUMMY_MODEL_SHAPE, precision}}}; - std::vector requestData{13.5, 0., 0, 0., 0., 0., 0., 0, 3., 67.}; - preparePredictRequest(request, inputsMeta, requestData); - request.mutable_model_name()->assign(modelName); - ASSERT_EQ(impl.ModelInfer(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - size_t dummysInTheGraph = 1; - checkDummyResponse("out", requestData, request, response, dummysInTheGraph, 1, modelName); -} - TEST_F(MediapipeTensorTest, DummyInfer) { const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); @@ -515,97 +464,6 @@ TEST_F(MediapipeTensorTest, DummyInfer) { checkDummyResponse("out", requestData, request, response, dummysInTheGraph, 1, modelName); } -TEST_F(MediapipeTfLiteTensorTest, DummyInfer) { - GTEST_SKIP() << "OVMS calculator doesn't handle TfLite on output. Only vector of TfLite" - << "OVMS deserialization & serialization of TfLiteTensors is not finished as well"; - const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); - KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); - ::KFSRequest request; - ::KFSResponse response; - const std::string modelName{"mpTfLiteTensorDummy"}; - request.Clear(); - response.Clear(); - // TfLite tensors don't hold batch size dimension so we send shape [10] instead of default dummy's [1, 10] - inputs_info_t inputsMeta{{"in", {{10}, precision}}}; - std::vector requestData{13.5, 0., 0, 0., 0., 0., 0., 0, 3., 67.}; - preparePredictRequest(request, inputsMeta, requestData); - request.mutable_model_name()->assign(modelName); - EXPECT_EQ(impl.ModelInfer(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - size_t dummysInTheGraph = 1; - checkDummyResponse("out", requestData, request, response, dummysInTheGraph, 1, modelName); -} - -// Incorrect KServe proto to TFTensor conversion -TEST_F(MediapipeTFTest, SendDummyInferMoreDataThanExpected) { - const std::string modelName{"mpTFDummy"}; - const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); - KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); - ::KFSRequest request; - ::KFSResponse response; - request.Clear(); - response.Clear(); - const size_t numElements = 50000; - inputs_info_t inputsMeta{{"in", {{1, numElements}, precision}}}; - std::vector requestData(numElements); - preparePredictRequest(request, inputsMeta, requestData); - request.mutable_model_name()->assign(modelName); - request.mutable_inputs(0)->set_shape(1, 1); // change only shape [1,numElements] to [1,1], keep data - ASSERT_NE(impl.ModelInfer(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); -} - -// Scalar in KServe proto to TFTensor conversion -TEST_F(MediapipeTFTest, DummyInferScalar) { - const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); - KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); - ::KFSRequest request; - ::KFSResponse response; - const std::string modelName{"mpTFScalar"}; - request.Clear(); - response.Clear(); - inputs_info_t inputsMeta{{"in", {{1}, precision}}}; - std::vector requestData{7.1f}; - preparePredictRequest(request, inputsMeta, requestData); - request.mutable_inputs(0)->clear_shape(); // imitate scalar - request.mutable_model_name()->assign(modelName); - ASSERT_EQ(impl.ModelInfer(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - ASSERT_EQ(response.model_name(), modelName); - ASSERT_EQ(response.outputs_size(), 1); - ASSERT_EQ(response.raw_output_contents_size(), 1); - ASSERT_EQ(response.outputs().begin()->name(), "out") << "Did not find:out"; - const auto& output_proto = *response.outputs().begin(); - std::string* content = response.mutable_raw_output_contents(0); - - ASSERT_EQ(content->size(), sizeof(float)); - ASSERT_EQ(output_proto.shape_size(), 0); -} - -// 0-data KServe proto to TFTensor conversion -TEST_F(MediapipeTFTest, DummyInferZeroData) { - const std::string modelName{"mpTFDummy"}; - const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); - KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); - ::KFSRequest request; - ::KFSResponse response; - request.Clear(); - response.Clear(); - inputs_info_t inputsMeta{{"in", {{1, 0}, precision}}}; - std::vector requestData; - preparePredictRequest(request, inputsMeta, requestData); - request.mutable_model_name()->assign(modelName); - ASSERT_EQ(impl.ModelInfer(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); - ASSERT_EQ(response.model_name(), modelName); - ASSERT_EQ(response.outputs_size(), 1); - ASSERT_EQ(response.raw_output_contents_size(), 1); - ASSERT_EQ(response.outputs().begin()->name(), "out") << "Did not find:out"; - const auto& output_proto = *response.outputs().begin(); - std::string* content = response.mutable_raw_output_contents(0); - - ASSERT_EQ(content->size(), 0); - ASSERT_EQ(output_proto.shape_size(), 2); - ASSERT_EQ(output_proto.shape(0), 1); - ASSERT_EQ(output_proto.shape(1), 0); -} - class MediapipeFlowDummyTest : public MediapipeFlowTest { public: void SetUp() { @@ -2799,22 +2657,6 @@ TEST_F(MediapipeSerialization, KFSResponse) { ASSERT_EQ(reinterpret_cast(mp_response.raw_output_contents().at(0).data())[0], 1.0f); } -TEST_F(MediapipeSerialization, TFTensor) { - tensorflow::Tensor response(TFSDataType::DT_FLOAT, {1}); - response.flat()(0) = 1.0f; - ::mediapipe::Packet packet = ::mediapipe::MakePacket(response); - ASSERT_EQ(onPacketReadySerializeImpl("1", "tf_response", "1", "tf_response", mediapipe_packet_type_enum::TFTENSOR, packet, mp_response), StatusCode::OK); - ASSERT_EQ(mp_response.id(), "1"); - ASSERT_EQ(mp_response.outputs(0).datatype(), "FP32"); - ASSERT_EQ(mp_response.outputs_size(), 1); - auto mp_output = mp_response.outputs(0); - ASSERT_EQ(mp_output.shape_size(), 1); - ASSERT_EQ(mp_output.shape(0), 1); - ASSERT_EQ(mp_response.raw_output_contents_size(), 1); - ASSERT_EQ(mp_response.raw_output_contents().at(0).size(), 4); - ASSERT_EQ(reinterpret_cast(mp_response.raw_output_contents().at(0).data())[0], 1.0f); -} - TEST_F(MediapipeSerialization, OVTensor) { std::vector data = {1.0f}; ov::element::Type type(ov::element::Type_t::f32); @@ -3740,51 +3582,6 @@ TEST_F(KFSGRPCContentFieldsSupportTestBytes, PyTensorBytesContentsCheckExpectedS } #endif -std::unordered_map> TYPE_TO_OVMS_PRECISION_TO_STATUS_TF_TENSOR{ - {typeid(float), {ovms::Precision::FP32, ovms::StatusCode::OK}}, - {typeid(uint64_t), {ovms::Precision::U64, ovms::StatusCode::OK}}, - {typeid(uint32_t), {ovms::Precision::U32, ovms::StatusCode::OK}}, - {typeid(uint16_t), {ovms::Precision::U16, ovms::StatusCode::OK}}, - {typeid(uint8_t), {ovms::Precision::U8, ovms::StatusCode::OK}}, - {typeid(int64_t), {ovms::Precision::I64, ovms::StatusCode::OK}}, - {typeid(int32_t), {ovms::Precision::I32, ovms::StatusCode::OK}}, - {typeid(int16_t), {ovms::Precision::I16, ovms::StatusCode::OK}}, - {typeid(int8_t), {ovms::Precision::I8, ovms::StatusCode::OK}}, - {typeid(bool), {ovms::Precision::BOOL, ovms::StatusCode::OK}}, - {typeid(double), {ovms::Precision::FP64, ovms::StatusCode::OK}}, - {typeid(void), {ovms::Precision::BIN, ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR}}}; - -TYPED_TEST(KFSGRPCContentFieldsSupportTest, TFTensorCheckExpectedStatusCode) { - const std::string pbtxtContentTFtensor = R"( - input_stream: "TFTENSOR:in" - output_stream: "TFTENSOR:out" - node { - calculator: "PassThroughCalculator" - input_stream: "TFTENSOR:in" - output_stream: "TFTENSOR:out" - } - )"; - this->CreateConfigAndPbtxt(pbtxtContentTFtensor); - char* argv[] = {(char*)"ovms", - (char*)"--config_path", - (char*)this->configFilePath.c_str(), - (char*)"--port", - (char*)this->port.c_str()}; - int argc = 5; - this->server.setShutdownRequest(0); - this->t = std::make_unique([&argc, &argv, this]() { - EXPECT_EQ(EXIT_SUCCESS, this->server.start(argc, argv)); - }); - // prepare data - std::vector data = prepareData(this->elemCount); - preparePredictRequest(this->request, - {{"in", {{1, 10}, TYPE_TO_OVMS_PRECISION_TO_STATUS_TF_TENSOR[typeid(TypeParam)].first}}}, - data, this->putDataInInputContents); - const std::string servableName{"mediapipeDummy"}; - this->request.mutable_model_name()->assign(servableName); - this->performInference(TYPE_TO_OVMS_PRECISION_TO_STATUS_TF_TENSOR[typeid(TypeParam)].second); -} - std::unordered_map> TYPE_TO_OVMS_PRECISION_TO_STATUS_MP_TENSOR{ {typeid(float), {ovms::Precision::FP32, ovms::StatusCode::OK}}, {typeid(uint64_t), {ovms::Precision::U64, ovms::StatusCode::INVALID_PRECISION}}, @@ -3901,19 +3698,6 @@ TYPED_TEST(KFSGRPCContentFieldsSupportTest, MPTensorInvalidContentSize) { this->performInvalidContentSizeTest(pbtxtContentMPTensor, TYPE_TO_STATUS_MP_TENSOR_INVALID_CONTENT_SIZE[typeid(TypeParam)]); } -TYPED_TEST(KFSGRPCContentFieldsSupportTest, TFTensorInvalidContentSize) { - const std::string pbtxtContentTFTensor = R"( - input_stream: "TFTENSOR:in" - output_stream: "TFTENSOR:out" - node { - calculator: "PassThroughCalculator" - input_stream: "TFTENSOR:in" - output_stream: "TFTENSOR:out" - } - )"; - this->performInvalidContentSizeTest(pbtxtContentTFTensor, ovms::StatusCode::INVALID_VALUE_COUNT); -} - INSTANTIATE_TEST_SUITE_P( Test, MediapipeFlowAddTest, diff --git a/src/test/pythonnode_test.cpp b/src/test/pythonnode_test.cpp index db6db588b5..437054faa4 100644 --- a/src/test/pythonnode_test.cpp +++ b/src/test/pythonnode_test.cpp @@ -45,7 +45,6 @@ #include "../server.hpp" #include "../shape.hpp" #include "../stringutils.hpp" -#include "src/tensorflow_type_utils.hpp" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include "mediapipe/framework/calculator_graph.h" diff --git a/src/test/stress_test_utils.hpp b/src/test/stress_test_utils.hpp index c76e1d1046..fc2de132b9 100644 --- a/src/test/stress_test_utils.hpp +++ b/src/test/stress_test_utils.hpp @@ -57,7 +57,6 @@ #include "test_with_temp_dir.hpp" #if (MEDIAPIPE_DISABLE == 0) #include "src/mediapipe_internal/mediapipegraphexecutor.hpp" -#include "src/tensorflow_type_utils.hpp" #endif using namespace ovms; diff --git a/third_party/mediapipe_calculators/BUILD b/third_party/mediapipe_calculators/BUILD index a4570a4daa..9a48658efc 100644 --- a/third_party/mediapipe_calculators/BUILD +++ b/third_party/mediapipe_calculators/BUILD @@ -70,11 +70,7 @@ cc_library( "@mediapipe//mediapipe/calculators/core:vector_size_calculator", "@mediapipe//mediapipe/calculators/core:packet_sequencer_calculator", "@mediapipe//mediapipe/calculators/core:merge_to_vector_calculator", - #TENSORFLOW LITE BELOW - "@mediapipe//mediapipe/calculators/tflite:ssd_anchors_calculator", - "@mediapipe//mediapipe/calculators/tflite:tflite_converter_calculator", - "@mediapipe//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator", - + "@mediapipe//mediapipe/calculators/util:annotation_overlay_calculator", "@mediapipe//mediapipe/calculators/util:detection_label_id_to_text_calculator", "@mediapipe//mediapipe/calculators/util:detections_to_render_data_calculator", diff --git a/third_party/tf_text/BUILD b/third_party/tf_text/BUILD deleted file mode 100644 index 5fbe4d5e7b..0000000000 --- a/third_party/tf_text/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright (c) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Empty BUILD so this is treated like a package. diff --git a/third_party/tf_text/tftext.patch b/third_party/tf_text/tftext.patch deleted file mode 100644 index ee1d89d3f5..0000000000 --- a/third_party/tf_text/tftext.patch +++ /dev/null @@ -1,38 +0,0 @@ -diff --git a/tensorflow_text/core/kernels/BUILD b/tensorflow_text/core/kernels/BUILD -index bdca365..5f5cdf6 100644 ---- a/tensorflow_text/core/kernels/BUILD -+++ b/tensorflow_text/core/kernels/BUILD -@@ -16,8 +16,7 @@ OSS_DEPS = [ - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", -- "@local_config_tf//:libtensorflow_framework", -- "@local_config_tf//:tf_header_lib", -+ "@org_tensorflow//tensorflow/core:tensorflow_opensource", - ] - - cc_library( -diff --git a/tensorflow_text/tftext.bzl b/tensorflow_text/tftext.bzl -index aa5e275..5eaff73 100644 ---- a/tensorflow_text/tftext.bzl -+++ b/tensorflow_text/tftext.bzl -@@ -44,8 +44,7 @@ def py_tf_text_library( - copts = [ "-pthread", ], - alwayslink = 1, - deps = cc_op_kernels + [ -- "@local_config_tf//:libtensorflow_framework", -- "@local_config_tf//:tf_header_lib", -+ "@org_tensorflow//tensorflow/core:tensorflow_opensource", - ], - ) - -@@ -55,8 +54,7 @@ def py_tf_text_library( - linkshared = 1, - deps = [ - ":" + library_name, -- "@local_config_tf//:libtensorflow_framework", -- "@local_config_tf//:tf_header_lib", -+ "@org_tensorflow//tensorflow/core:tensorflow_opensource", - ], - ) - From c7317a63067e278cb9906d0e702efdac6b75059b Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 17 Sep 2026 10:02:26 +0200 Subject: [PATCH 02/10] style --- .../ovms/openvinoinferencecalculatoroptions.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc index 90f9ca18d8..d77afd62b9 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc @@ -42,7 +42,7 @@ static bool ValidateOrderLists(std::set calculatorTags, const googl for (const auto& supportedVectorTag : supportedVectorTags) { if (startsWith(inputType, supportedVectorTag)) { if (order_list.size() < 1) { - LOG(ERROR) << "OpenVINOInferenceCalculator GetContract error. Order list is requiered for vector types: " << inputType; + LOG(ERROR) << "OpenVINOInferenceCalculator GetContract error. Order list is required for vector types: " << inputType; return false; } } From 9d2d26e3e3e7000981e5a8fc6f22b8add8d9078d Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 17 Sep 2026 11:02:49 +0200 Subject: [PATCH 03/10] style --- src/mediapipe_calculators/ovms/modelapiovmsadapter.cc | 3 ++- .../ovms/openvinoinferencecalculator.cc | 10 +++++----- .../ovms/openvinoinferencecalculator.h | 6 +++--- .../ovms/openvinoinferencecalculatoroptions.cc | 4 ++-- .../ovms/openvinoinferencecalculatoroptions.h | 1 + .../ovms/openvinoinferencedumputils.cc | 5 +++-- .../ovms/openvinoinferencedumputils.h | 1 + .../ovms/openvinoinferenceutils.cc | 1 + .../ovms/openvinoinferenceutils.h | 1 + .../ovms/openvinomodelserversessioncalculator.cc | 5 +++-- .../ovms/openvinomodelserversessioncalculator.h | 8 +++++--- 11 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/mediapipe_calculators/ovms/modelapiovmsadapter.cc b/src/mediapipe_calculators/ovms/modelapiovmsadapter.cc index 76aafe3476..95d221e33c 100644 --- a/src/mediapipe_calculators/ovms/modelapiovmsadapter.cc +++ b/src/mediapipe_calculators/ovms/modelapiovmsadapter.cc @@ -15,9 +15,10 @@ //***************************************************************************** #include "modelapiovmsadapter.hpp" #if (OVMS_DUMP_TO_FILE == 1) -#include "openvinoinferencedumputils.h" +#include "src/mediapipe_calculators/ovms/openvinoinferencedumputils.h" #endif #include +#include #include #include #include diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc index f24b1f37c5..bc2ed896be 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc @@ -32,12 +32,12 @@ #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port/canonical_errors.h" -#include "openvinoinferencecalculator.h" -#include "openvinoinferencecalculatoroptions.h" -#include "openvinoinferenceutils.h" +#include "src/mediapipe_calculators/ovms/openvinoinferencecalculator.h" +#include "src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h" +#include "src/mediapipe_calculators/ovms/openvinoinferenceutils.h" #include "src/mediapipe_calculators/ovms/openvinoinferencecalculator.pb.h" #if (OVMS_DUMP_TO_FILE == 1) -#include "openvinoinferencedumputils.h" +#include "src/mediapipe_calculators/ovms/openvinoinferencedumputils.h" #endif #pragma GCC diagnostic pop namespace mediapipe { @@ -275,7 +275,7 @@ absl::Status OpenVINOInferenceCalculator::Process(CalculatorContext* cc) { try { if (startsWith(tag, OVTENSORS_TAG)) { - DESERIALIZE_TENSORS(ov::Tensor, ); + DESERIALIZE_TENSORS(ov::Tensor,); // NOLINT(whitespace/comma) } else if (startsWith(tag, MPTENSORS_TAG)) { DESERIALIZE_TENSORS(Tensor, convertMPTensor2OVTensor); } else if (startsWith(tag, OVTENSOR_TAG)) { diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.h b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.h index 4963239e04..c31892460b 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.h +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.h @@ -35,8 +35,8 @@ class OpenVINOInferenceCalculator : public CalculatorBase { public: static absl::Status GetContract(CalculatorContract* cc); - absl::Status Close(CalculatorContext* cc) override final; - absl::Status Open(CalculatorContext* cc) override final; - absl::Status Process(CalculatorContext* cc) override final; + absl::Status Close(CalculatorContext* cc) final; + absl::Status Open(CalculatorContext* cc) final; + absl::Status Process(CalculatorContext* cc) final; }; } // namespace mediapipe diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc index d77afd62b9..518df214fe 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.cc @@ -25,8 +25,8 @@ #include "mediapipe/framework/calculator_contract.h" #include "src/mediapipe_calculators/ovms/openvinoinferencecalculator.pb.h" #pragma GCC diagnostic pop -#include "openvinoinferencecalculatoroptions.h" -#include "openvinoinferenceutils.h" +#include "src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h" +#include "src/mediapipe_calculators/ovms/openvinoinferenceutils.h" namespace mediapipe { diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h index 2d4bda3d76..a55cd01b92 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculatoroptions.h @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#pragma once #include #include #include diff --git a/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc index 75b4a47165..6e3dd836d6 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc +++ b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc @@ -24,7 +24,7 @@ #include -#include "openvinoinferencedumputils.h" +#include "src/mediapipe_calculators/ovms/openvinoinferencedumputils.h" namespace mediapipe { @@ -107,7 +107,8 @@ static void writeToFile(std::stringstream& stream, std::string name) { static std::string getTimestampString() { auto rawtime = std::make_unique(); time(rawtime.get()); - struct tm* timeinfo = localtime(rawtime.get()); + struct tm timeinfoBuf; + struct tm* timeinfo = localtime_r(rawtime.get(), &timeinfoBuf); auto start = std::chrono::system_clock::now(); std::stringstream timestampStream; timestampStream << timeinfo->tm_year << "_" << timeinfo->tm_mon << "_" << timeinfo->tm_mday << "_"; diff --git a/src/mediapipe_calculators/ovms/openvinoinferencedumputils.h b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.h index fe34d9654b..2b24e081b4 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencedumputils.h +++ b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.h @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#pragma once #include #include diff --git a/src/mediapipe_calculators/ovms/openvinoinferenceutils.cc b/src/mediapipe_calculators/ovms/openvinoinferenceutils.cc index 41e580a03e..1023b4f32e 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferenceutils.cc +++ b/src/mediapipe_calculators/ovms/openvinoinferenceutils.cc @@ -20,6 +20,7 @@ #include #include "ovms.h" // NOLINT +#include "src/mediapipe_calculators/ovms/openvinoinferenceutils.h" namespace mediapipe { // Function from ovms/src/string_utils.h diff --git a/src/mediapipe_calculators/ovms/openvinoinferenceutils.h b/src/mediapipe_calculators/ovms/openvinoinferenceutils.h index fc1cd385f7..ca614baa8f 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferenceutils.h +++ b/src/mediapipe_calculators/ovms/openvinoinferenceutils.h @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#pragma once #include #include diff --git a/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.cc b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.cc index c89f0cf746..0bb1733240 100644 --- a/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.cc +++ b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.cc @@ -13,11 +13,12 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#include "openvinomodelserversessioncalculator.h" +#include "src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.h" #include #include #include +#include #include #include #include @@ -35,7 +36,7 @@ #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/canonical_errors.h" #include "modelapiovmsadapter.hpp" -#include "openvinoinferenceutils.h" +#include "src/mediapipe_calculators/ovms/openvinoinferenceutils.h" #include "src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.pb.h" #pragma GCC diagnostic pop namespace mediapipe { diff --git a/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.h b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.h index fbbb9f5e73..d044e33cef 100644 --- a/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.h +++ b/src/mediapipe_calculators/ovms/openvinomodelserversessioncalculator.h @@ -13,11 +13,13 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#pragma once #include #include #include #include #include +#include #include #include @@ -42,9 +44,9 @@ class OpenVINOModelServerSessionCalculator : public CalculatorBase { public: static absl::Status GetContract(CalculatorContract* cc); - absl::Status Close(CalculatorContext* cc) override final; - absl::Status Open(CalculatorContext* cc) override final; - absl::Status Process(CalculatorContext* cc) override final; + absl::Status Close(CalculatorContext* cc) final; + absl::Status Open(CalculatorContext* cc) final; + absl::Status Process(CalculatorContext* cc) final; static OVMS_LogLevel OvmsLogLevel; static const char* OvmsLogLevelEnv; }; From abfbb6d5e5b8ab4f85ac148ef187e7e259b3f168 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 17 Sep 2026 11:13:32 +0200 Subject: [PATCH 04/10] style1 --- src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc index bc2ed896be..3d28aba529 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc @@ -275,7 +275,7 @@ absl::Status OpenVINOInferenceCalculator::Process(CalculatorContext* cc) { try { if (startsWith(tag, OVTENSORS_TAG)) { - DESERIALIZE_TENSORS(ov::Tensor,); // NOLINT(whitespace/comma) + DESERIALIZE_TENSORS(ov::Tensor, ); // NOLINT(whitespace/comma) } else if (startsWith(tag, MPTENSORS_TAG)) { DESERIALIZE_TENSORS(Tensor, convertMPTensor2OVTensor); } else if (startsWith(tag, OVTENSOR_TAG)) { From a848eb4d898390d37dcac50ec941c26b144ba7c7 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 17 Sep 2026 11:37:09 +0200 Subject: [PATCH 05/10] style2 --- src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc index 3d28aba529..fa497c62dd 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc +++ b/src/mediapipe_calculators/ovms/openvinoinferencecalculator.cc @@ -275,7 +275,7 @@ absl::Status OpenVINOInferenceCalculator::Process(CalculatorContext* cc) { try { if (startsWith(tag, OVTENSORS_TAG)) { - DESERIALIZE_TENSORS(ov::Tensor, ); // NOLINT(whitespace/comma) + DESERIALIZE_TENSORS(ov::Tensor, ); // NOLINT } else if (startsWith(tag, MPTENSORS_TAG)) { DESERIALIZE_TENSORS(Tensor, convertMPTensor2OVTensor); } else if (startsWith(tag, OVTENSOR_TAG)) { From c272ef2411afbabc35fe846ce6acb5c41fe0c093 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 17 Sep 2026 12:08:53 +0200 Subject: [PATCH 06/10] win-build-fix --- .../ovms/openvinoinferencedumputils.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc index 6e3dd836d6..bddedbfec6 100644 --- a/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc +++ b/src/mediapipe_calculators/ovms/openvinoinferencedumputils.cc @@ -108,7 +108,12 @@ static std::string getTimestampString() { auto rawtime = std::make_unique(); time(rawtime.get()); struct tm timeinfoBuf; - struct tm* timeinfo = localtime_r(rawtime.get(), &timeinfoBuf); +#ifdef _WIN32 + localtime_s(&timeinfoBuf, rawtime.get()); +#else + localtime_r(rawtime.get(), &timeinfoBuf); +#endif + struct tm* timeinfo = &timeinfoBuf; auto start = std::chrono::system_clock::now(); std::stringstream timestampStream; timestampStream << timeinfo->tm_year << "_" << timeinfo->tm_mon << "_" << timeinfo->tm_mday << "_"; From 367d175806c5d8d6e7cbf21c6341413bd6e1d4aa Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 17 Sep 2026 15:37:45 +0200 Subject: [PATCH 07/10] drop removed calculators --- src/test/mediapipeflow_test.cpp | 139 +------------------------------- 1 file changed, 2 insertions(+), 137 deletions(-) diff --git a/src/test/mediapipeflow_test.cpp b/src/test/mediapipeflow_test.cpp index e0d7f6d069..02177b1d89 100644 --- a/src/test/mediapipeflow_test.cpp +++ b/src/test/mediapipeflow_test.cpp @@ -3723,7 +3723,6 @@ TEST(WhitelistRegistered, InputStreamHandlers) { "BarrierInputStreamHandler", "DefaultInputStreamHandler", "EarlyCloseInputStreamHandler", - "FixedSizeInputStreamHandler", "ImmediateInputStreamHandler", "MuxInputStreamHandler", "SyncSetInputStreamHandler", @@ -3746,11 +3745,7 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "AddOne3CycleIterationsTestCalculator", "AddOneSingleStreamTestCalculator", "AddSidePacketToSingleStreamTestCalculator", - "AlignmentPointsRectsCalculator", "AnnotationOverlayCalculator", - "AnomalyCalculator", - "AnomalySerializationCalculator", - "AssociationNormRectCalculator", "BeginLoopDetectionCalculator", "BeginLoopFloatCalculator", "BeginLoopGpuBufferCalculator", @@ -3759,10 +3754,8 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "BeginLoopIntCalculator", "BeginLoopMatrixCalculator", "BeginLoopMatrixVectorCalculator", - "BeginLoopModelApiDetectionCalculator", "BeginLoopNormalizedLandmarkListVectorCalculator", "BeginLoopNormalizedRectCalculator", - "BeginLoopRectanglePredictionCalculator", "BeginLoopStringCalculator", "BeginLoopTensorCalculator", "BeginLoopUint64tCalculator", @@ -3771,10 +3764,7 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "CallbackCalculator", "CallbackPacketCalculator", "CallbackWithHeaderCalculator", - "ClassificationCalculator", - "ClassificationListVectorHasMinSizeCalculator", "ClassificationListVectorSizeCalculator", - "ClassificationSerializationCalculator", "ClipDetectionVectorSizeCalculator", "ClipNormalizedRectVectorSizeCalculator", "ColorConvertCalculator", @@ -3799,30 +3789,12 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "ConcatenateUInt64VectorCalculator", "ConstantSidePacketCalculator", "CountingSourceCalculator", - "CropCalculator", "DefaultSidePacketCalculator", "DequantizeByteArrayCalculator", - "DetectionCalculator", - "DetectionClassificationCombinerCalculator", - "DetectionClassificationResultCalculator", - "DetectionClassificationSerializationCalculator", - "DetectionExtractionCalculator", "DetectionLabelIdToTextCalculator", - "DetectionLetterboxRemovalCalculator", - "DetectionProjectionCalculator", - "DetectionSegmentationCombinerCalculator", - "DetectionSegmentationResultCalculator", - "DetectionSegmentationSerializationCalculator", - "DetectionSerializationCalculator", - "DetectionsToRectsCalculator", "DetectionsToRenderDataCalculator", "EmbeddingsCalculatorOV", "RerankCalculatorOV", - "EmptyLabelCalculator", - "EmptyLabelClassificationCalculator", - "EmptyLabelDetectionCalculator", - "EmptyLabelRotatedDetectionCalculator", - "EmptyLabelSegmentationCalculator", "EndLoopAffineMatrixCalculator", "EndLoopBooleanCalculator", "EndLoopClassificationListCalculator", @@ -3834,12 +3806,8 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "EndLoopImageSizeCalculator", "EndLoopLandmarkListVectorCalculator", "EndLoopMatrixCalculator", - "EndLoopModelApiDetectionClassificationCalculator", - "EndLoopModelApiDetectionSegmentationCalculator", "EndLoopNormalizedLandmarkListVectorCalculator", "EndLoopNormalizedRectCalculator", - "EndLoopPolygonPredictionsCalculator", - "EndLoopRectanglePredictionsCalculator", "EndLoopRenderDataCalculator", "EndLoopTensorCalculator", "EndLoopTfLiteTensorCalculator", @@ -3849,12 +3817,10 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "ExceptionDuringGetContractCalculator", "ExceptionDuringOpenCalculator", "ExceptionDuringProcessCalculator", - "FaceLandmarksToRenderDataCalculator", "FeatureDetectorCalculator", "FlowLimiterCalculator", "FlowPackagerCalculator", "FlowToImageCalculator", - "FromImageCalculator", "GateCalculator", "GetClassificationListVectorItemCalculator", "GetDetectionVectorItemCalculator", @@ -3863,9 +3829,6 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "GetNormalizedRectVectorItemCalculator", "GetRectVectorItemCalculator", "GraphProfileCalculator", - "HandDetectionsFromPoseToRectsCalculator", - "HandLandmarksToRectCalculator", - "HttpSerializationCalculator", "ImageCloneCalculator", "ImageCroppingCalculator", "ImagePropertiesCalculator", @@ -3873,20 +3836,8 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "ImageTransformationCalculator", "ImmediateMuxCalculator", "ImageGenCalculator", - "InferenceCalculatorCpu", "InputSidePacketUserTestCalc", - "InstanceSegmentationCalculator", - "InverseMatrixCalculator", - "IrisToRenderDataCalculator", - "KeypointDetectionCalculator", - "LandmarkLetterboxRemovalCalculator", "LandmarkListVectorSizeCalculator", - "LandmarkProjectionCalculator", - "LandmarkVisibilityCalculator", - "LandmarksRefinementCalculator", - "LandmarksSmoothingCalculator", - "LandmarksToDetectionCalculator", - "LandmarksToRenderDataCalculator", "LongLoadingCalculator", "MakePairCalculator", "MatrixMultiplyCalculator", @@ -3897,8 +3848,6 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "MergeDetectionsToVectorCalculator", "MergeGpuBuffersToVectorCalculator", "MergeImagesToVectorCalculator", - "ModelInferHttpRequestCalculator", - "ModelInferRequestImageCalculator", "MotionAnalysisCalculator", "MultipartAcceptingCalculator", "MuxCalculator", @@ -3906,9 +3855,6 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "NoOutputStreamsProducedCalculator", "NonMaxSuppressionCalculator", "NonZeroCalculator", - "NormalizedLandmarkListVectorHasMinSizeCalculator", - "NormalizedRectVectorHasMinSizeCalculator", - "OverlayCalculator", "OVMSOVCalculator", "OVMSTestImageInputPassthroughCalculator", "OVMSTestKFSPassCalculator", @@ -3917,12 +3863,8 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "OpenCvPutTextCalculator", "OpenCvVideoDecoderCalculator", "OpenCvVideoEncoderCalculator", - "OpenVINOConverterCalculator", - "OpenVINOInferenceAdapterCalculator", "OpenVINOInferenceCalculator", "OpenVINOModelServerSessionCalculator", - "OpenVINOTensorsToClassificationCalculator", - "OpenVINOTensorsToDetectionsCalculator", #ifndef _WIN32 // TODO windows: stdc++20 required "PacketClonerCalculator", #endif @@ -3935,22 +3877,9 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "PassThroughCalculator", "PreviousLoopbackCalculator", "QuantizeFloatVectorCalculator", - "RectToRenderDataCalculator", - "RectToRenderScaleCalculator", - "RectTransformationCalculator", - "RefineLandmarksFromHeatmapCalculator", - "ResourceProviderCalculator", - "RoiTrackingCalculator", - "RotatedDetectionCalculator", - "RotatedDetectionSerializationCalculator", "RoundRobinDemuxCalculator", - "SegmentationCalculator", - "SegmentationSerializationCalculator", "SegmentationSmoothingCalculator", "SequenceShiftCalculator", - "SerializationCalculator", - "SetLandmarkVisibilityCalculator", - "SidePacketToStreamCalculator", "S2tCalculator", "T2sCalculator", "SplitAffineMatrixVectorCalculator", @@ -3968,7 +3897,6 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "SplitTensorVectorCalculator", "SplitTfLiteTensorVectorCalculator", "SplitUint64tVectorCalculator", - "SsdAnchorsCalculator", "StreamToSidePacketCalculator", "StringToInt32Calculator", "StringToInt64Calculator", @@ -3976,83 +3904,20 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { "StringToUint32Calculator", "StringToUint64Calculator", "StringToUintCalculator", - "SwitchDemuxCalculator", - "SwitchMuxCalculator", - "TensorsToClassificationCalculator", - "TensorsToDetectionsCalculator", - "TensorsToFloatsCalculator", - "TensorsToLandmarksCalculator", - "TensorsToSegmentationCalculator", - "TfLiteConverterCalculator", - "TfLiteCustomOpResolverCalculator", - "TfLiteInferenceCalculator", - "TfLiteModelCalculator", - "TfLiteTensorsToDetectionsCalculator", - "TfLiteTensorsToFloatsCalculator", - "TfLiteTensorsToLandmarksCalculator", - "ThresholdingCalculator", - "ToImageCalculator", "TrackedDetectionManagerCalculator", #ifndef _WIN32 // TODO windows: 'opencv2/optflow.hpp': No such file - will be available with opencv cmake on windows "Tvl1OpticalFlowCalculator", #endif "TwoInputCalculator", - "UpdateFaceLandmarksCalculator", "VideoPreStreamCalculator", - "VisibilityCopyCalculator", - "VisibilitySmoothingCalculator", "WarpAffineCalculator", - "WarpAffineCalculatorCpu", - "WorldLandmarkProjectionCalculator" }); + "WarpAffineCalculatorCpu" }); ASSERT_THAT(mediapipe::CalculatorBaseRegistry::GetRegisteredNames(), UnorderedElementsAreArray(expected)) << readableSetError(mediapipe::CalculatorBaseRegistry::GetRegisteredNames(), expected); } TEST(WhitelistRegistered, MediapipeSubgraphList) { - std::unordered_set expected({"FaceDetection", - "FaceDetectionFrontDetectionToRoi", - "FaceDetectionFrontDetectionsToRoi", - "FaceDetectionShortRange", - "FaceDetectionShortRangeByRoiCpu", - "FaceDetectionShortRangeCpu", - "FaceLandmarkCpu", - "FaceLandmarkFrontCpu", - "FaceLandmarkLandmarksToRoi", - "FaceLandmarksFromPoseCpu", - "FaceLandmarksFromPoseToRecropRoi", - "FaceLandmarksModelLoader", - "FaceLandmarksToRoi", - "FaceTracking", - "HandLandmarkCpu", - "HandLandmarkModelLoader", - "HandLandmarksFromPoseCpu", - "HandLandmarksFromPoseToRecropRoi", - "HandLandmarksLeftAndRightCpu", - "HandLandmarksToRoi", - "HandRecropByRoiCpu", - "HandTracking", - "HandVisibilityFromHandLandmarksFromPose", - "HandWristForPose", - "HolisticLandmarkCpu", - "HolisticTrackingToRenderData", - "InferenceCalculator", - "IrisLandmarkCpu", - "IrisLandmarkLandmarksToRoi", - "IrisLandmarkLeftAndRightCpu", - "IrisRendererCpu", - "PoseDetectionCpu", - "PoseDetectionToRoi", - "PoseLandmarkByRoiCpu", - "PoseLandmarkCpu", - "PoseLandmarkFiltering", - "PoseLandmarkModelLoader", - "PoseLandmarksAndSegmentationInverseProjection", - "PoseLandmarksToRoi", - "PoseSegmentationFiltering", - "SwitchContainer", - "TensorsToFaceLandmarks", - "TensorsToFaceLandmarksWithAttention", - "TensorsToPoseLandmarksAndSegmentation"}); + std::unordered_set expected({}); ASSERT_THAT(mediapipe::SubgraphRegistry::GetRegisteredNames(), UnorderedElementsAreArray(expected)) << readableSetError(mediapipe::SubgraphRegistry::GetRegisteredNames(), expected); } From 94dfe8d169041a02d5db4482e06f3fd00d3bf7da Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Fri, 18 Sep 2026 11:06:08 +0200 Subject: [PATCH 08/10] experimental version --- .bazelversion | 2 +- BUILD.bazel | 1 - Dockerfile.redhat | 2 +- Dockerfile.ubuntu | 2 +- WORKSPACE | 218 ++++++------ ci/lib_search.py | 4 - common_settings.bzl | 5 + external/tf_2.18_logging.patch | 315 ------------------ external/tf_nsync_chrono.patch | 31 -- src/audio/speech_to_text/s2t_servable.hpp | 5 +- src/dags/dl_node.cpp | 1 + src/embeddings/embeddings_api.cpp | 3 +- src/filesystem/azurestorage.cpp | 2 +- src/http_rest_api_handler.cpp | 11 +- src/image_gen/imagegenutils.cpp | 2 +- src/kfs_frontend/kfs_graph_executor_impl.cpp | 6 +- src/kfs_frontend/kfs_utils.hpp | 4 +- src/kfserving_api/BUILD | 38 ++- .../mediapipe_graph_executor_interface.hpp | 4 +- .../mediapipegraphexecutor.hpp | 2 +- src/test/deserialization_tests.cpp | 4 +- .../audio_decoding_processor_test.cpp | 4 +- .../input_processing_integration_test.cpp | 4 +- src/test/ov_utils_test.cpp | 8 +- src/test/ovinferrequestqueue_test.cpp | 2 +- src/test/serialization_tests.cpp | 10 +- src/test/test_models.hpp | 20 +- src/test/text2image_test.cpp | 2 +- third_party/BUILD | 2 +- third_party/absl/BUILD | 17 + third_party/absl/absl_constexpr_fix.patch | 41 +++ third_party/grpc/BUILD | 17 + .../grpc/grpc_missing_algorithm_include.patch | 23 ++ third_party/mediapipe/BUILD | 17 + third_party/mediapipe/ovms_strip.diff | 250 ++++++++++++++ third_party/mediapipe_calculators/BUILD | 34 +- third_party/model_api/BUILD | 17 + third_party/model_api/model_api.bzl | 158 +++++++++ .../model_api/model_api_json_archive.patch | 19 ++ third_party/protobuf/BUILD | 17 + .../protobuf/com_google_protobuf_fixes.diff | 92 +++++ 41 files changed, 868 insertions(+), 548 deletions(-) delete mode 100644 external/tf_2.18_logging.patch delete mode 100644 external/tf_nsync_chrono.patch create mode 100644 third_party/absl/BUILD create mode 100644 third_party/absl/absl_constexpr_fix.patch create mode 100644 third_party/grpc/BUILD create mode 100644 third_party/grpc/grpc_missing_algorithm_include.patch create mode 100644 third_party/mediapipe/BUILD create mode 100644 third_party/mediapipe/ovms_strip.diff create mode 100644 third_party/model_api/BUILD create mode 100644 third_party/model_api/model_api.bzl create mode 100644 third_party/model_api/model_api_json_archive.patch create mode 100644 third_party/protobuf/BUILD create mode 100644 third_party/protobuf/com_google_protobuf_fixes.diff diff --git a/.bazelversion b/.bazelversion index f3b5af39e4..815da58b7a 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.1.1 +7.4.1 diff --git a/BUILD.bazel b/BUILD.bazel index 17b30de20c..e8396d7690 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -36,7 +36,6 @@ cc_library( "@minitrace//:trace", "@com_github_grpc_grpc//:grpc++", "@com_github_tencent_rapidjson//:rapidjson", - "@org_tensorflow//tensorflow/core:framework", "@com_github_gabime_spdlog//:spdlog", "@com_github_jarro2783_cxxopts//:cxxopts", "//third_party:openvino", diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 9799c08bb9..2f34f50859 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -170,7 +170,7 @@ RUN wget -q https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packag RUN python3.12 --version && python3.12 -m pip install "numpy<2.0.0" "Jinja2==3.1.6" "MarkupSafe==3.0.2" --no-cache-dir # Set up Bazel -ENV BAZEL_VERSION 6.1.1 +ENV BAZEL_VERSION 7.4.1 WORKDIR /bazel RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index b8ef46a623..a2a8b3a661 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -159,7 +159,7 @@ ENV HDDL_INSTALL_DIR=/opt/intel/openvino/deployment_tools/inference_engine/exter ENV TF_SYSTEM_LIBS="curl" # Set up Bazel -ENV BAZEL_VERSION 6.1.1 +ENV BAZEL_VERSION 7.4.1 WORKDIR /bazel RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ diff --git a/WORKSPACE b/WORKSPACE index 1ecba9f0b1..66b9d8badf 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -42,19 +42,47 @@ http_archive( http_archive( name = "bazel_skylib", - sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", + sha256 = "37cdfbc6faefea94f7b37760a305c98c08981116c2bc9e821e3b423221fad8c8", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.9.2/bazel-skylib-1.9.2.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.9.2/bazel-skylib-1.9.2.tar.gz", ], ) load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") bazel_skylib_workspace() load("@bazel_skylib//lib:versions.bzl", "versions") -versions.check(minimum_bazel_version = "6.0.0") +versions.check(minimum_bazel_version = "7.0.0") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +# platforms bumped to include ppc64le (needed by rules_go under bazel 7 / grpc_extra_deps). +http_archive( + name = "platforms", + sha256 = "3384eb1c30762704fbe38e440204e114154086c8fc8a8c2e3e28441028c019a8", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/1.0.0/platforms-1.0.0.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/1.0.0/platforms-1.0.0.tar.gz", + ], +) + +# bazel_features - required by newer rules_cc / mediapipe upstream. +http_archive( + name = "bazel_features", + sha256 = "5ac743bf5f05d88e84962e978811f2524df09602b789c92cf7ae2111ecdeda94", + strip_prefix = "bazel_features-1.14.0", + url = "https://github.com/bazel-contrib/bazel_features/releases/download/v1.14.0/bazel_features-v1.14.0.tar.gz", +) +load("@bazel_features//:deps.bzl", "bazel_features_deps") +bazel_features_deps() + +# Newer rules_cc required by upstream mediapipe (@rules_cc//cc:cc_library.bzl, cc_binary.bzl). +http_archive( + name = "rules_cc", + sha256 = "b8b918a85f9144c01f6cfe0f45e4f2838c7413961a8ff23bc0c6cdf8bb07a3b6", + strip_prefix = "rules_cc-0.1.5", + url = "https://github.com/bazelbuild/rules_cc/releases/download/0.1.5/rules_cc-0.1.5.tar.gz", +) + http_archive( name = "rules_python", sha256 = "0cc05ddb27614baecace068986931e2a6e9f69114e6115fc5dc58250faf56e0f", @@ -66,22 +94,25 @@ load("@rules_python//python:repositories.bzl", "py_repositories") py_repositories() -# ABSL on 2023-10-18 -# Needed for MP -# https://github.com/google-ai-edge/mediapipe/commit/743cdb747332efdfb43338d92aa6349acc40a06a -# patch for static_assert(ValidateAsciiCasefold() == 0, "error in case conversion"); -# needs to be before MP & TF -git_repository( +# ABSL pinned to a release with `absl/status/status_macros.h` (required by +# mediapipe upstream) and compatible with protobuf 6.31 / gRPC 1.74. +http_archive( name = "com_google_absl", - remote = "https://github.com/abseil/abseil-cpp", - commit = "9687a8ea750bfcddf790372093245a1d041b21a3", # MP image buildable original MP - patches = [ - "@mediapipe//third_party:com_google_absl_windows_patch.diff", - "abseil_gcc_8.5_constant_expression.patch", - ], - patch_args = [ - "-p1", - ], + sha256 = "6e1aee535473414164bf83e4ebc40240dec71a4701f8a642d906e95bea1aea0c", + strip_prefix = "abseil-cpp-20260526.0", + urls = ["https://github.com/abseil/abseil-cpp/archive/refs/tags/20260526.0.tar.gz"], + patches = ["@ovms//third_party/absl:absl_constexpr_fix.patch"], + patch_args = ["-p1"], + patch_tool = "patch", +) + +# Pre-declare `com_google_googletest` (satisfies protobuf_deps's existing-rule +# check) so it does not pull in a second abseil under the name `@abseil-cpp`. +http_archive( + name = "com_google_googletest", + sha256 = "65fab701d9829d38cb77c14acdc431d2108bfdbf8979e40eb8ae567edf10b27c", + strip_prefix = "googletest-1.17.0", + urls = ["https://github.com/google/googletest/archive/refs/tags/v1.17.0.tar.gz"], ) http_archive( @@ -122,8 +153,9 @@ new_local_repository( build_file = "@//third_party/boringssl:BUILD", ) +# `@curl` is used by google_cloud_cpp (via repo_mapping) and OVMS itself. new_local_repository( - name = "linux_curl", + name = "curl", path = "/usr/", build_file_content = """ cc_library( @@ -161,23 +193,32 @@ cc_library( ########################################################### Mediapipe http_archive( name = "com_google_protobuf", - sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422", - strip_prefix = "protobuf-3.19.1", - urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"], - patches = [ - "@mediapipe//third_party:com_google_protobuf_fixes.diff" - ], - patch_args = [ - "-p1", - ], + sha256 = "597071a340acc5346494c119ba3a541825c3f81071fc783521b24e29a485d60f", + strip_prefix = "protobuf-6.31.1", + urls = ["https://github.com/protocolbuffers/protobuf/archive/refs/tags/v6.31.1.tar.gz"], + repo_mapping = {"@abseil-cpp": "@com_google_absl"}, ) -################################### Official/forked mediapipe repository ######### +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") +protobuf_deps() + +load("@rules_java//java:rules_java_deps.bzl", "rules_java_dependencies") +rules_java_dependencies() +load("@rules_java//java:repositories.bzl", "rules_java_toolchains") +rules_java_toolchains() + +################################### Upstream mediapipe repository ############### #### Will be used on feature release git_repository( name = "mediapipe", - remote = "https://github.com/openvinotoolkit/mediapipe", - commit = "b27f0ca7d4fe36175a9f725844cb64b3d3e18959", # top of mediapipe main branch as of 13.09.2026 + remote = "https://github.com/google-ai-edge/mediapipe", + commit = "2bce9dd15fa45f267c9e5f77086997c984a9f107", # top of mediapipe master branch as of 17.09.2026 + patches = [ + "@ovms//third_party/mediapipe:ovms_strip.diff", + "@ovms//third_party/mediapipe:ovms_no_litert.diff", + ], + patch_args = ["-p1"], + patch_tool = "patch", ) # DEV mediapipe 1 source - adjust local repository path for build @@ -385,91 +426,27 @@ cc_library( """, ) -# We need to override upb due to false positive stringop-truncation warning -# second patch is needed & copied from TF -http_archive( - name = "upb", - sha256 = "61d0417abd60e65ed589c9deee7c124fe76a4106831f6ad39464e1525cef1454", - strip_prefix = "upb-9effcbcb27f0a665f9f345030188c0b291e32482", - patches = [ - "upb_platform_fix.patch", - "upb_warning_turn_off.patch" - ], - patch_args = [ - "-p1", +# grpc must be defined before grpc_deps() below so this version is picked up. +http_archive( # 1.74.1 + name = "com_github_grpc_grpc", + urls = [ + "https://github.com/grpc/grpc/archive/893bdadd56dbb75fb156175afdaa2b0d47e1c15b.tar.gz", ], - urls = ["https://github.com/protocolbuffers/upb/archive/9effcbcb27f0a665f9f345030188c0b291e32482.tar.gz"], + strip_prefix = "grpc-893bdadd56dbb75fb156175afdaa2b0d47e1c15b", + patches = ["@ovms//third_party/grpc:grpc_missing_algorithm_include.patch"], + patch_args = ["-p1"], + repo_mapping = {"@abseil-cpp": "@com_google_absl"}, ) -# TensorFlow repo should always go after the other external dependencies. -# TF on 2024-09-24 same as in Mediapipe -_TENSORFLOW_GIT_COMMIT = "5329ec8dd396487982ef3e743f98c0195af39a6b" -_TENSORFLOW_SHA256 = "eb1f8d740d59ea3dee91108ab1fc19d91c4e9ac2fd17d9ab86d865c3c43d81c9" +# rules_pkg was previously provided transitively via TensorFlow's workspace macros. http_archive( - name = "org_tensorflow", + name = "rules_pkg", urls = [ - "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, - ], - patches = [ - "@mediapipe//third_party:org_tensorflow_c_api_experimental.diff", - "@mediapipe//third_party:org_tensorflow_custom_ops.diff", - "@mediapipe//third_party:org_tensorflow_objc_build_fixes.diff", - "tf_2.18_logging.patch", - "tf_nsync_chrono.patch", - ], - patch_args = [ - "-p1", + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", ], - strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT, - sha256 = _TENSORFLOW_SHA256, - repo_mapping = {"@curl" : "@curl"} -) - -# Initialize TensorFlow's external dependencies. -load("@org_tensorflow//tensorflow:workspace3.bzl", "workspace") -workspace() -# Initialize hermetic Python -load("@org_tensorflow//third_party/xla/third_party/py:python_init_rules.bzl", "python_init_rules") -python_init_rules() - -load("@org_tensorflow//third_party/xla/third_party/py:python_init_repositories.bzl", "python_init_repositories") -python_init_repositories( - default_python_version = "system", - local_wheel_dist_folder = "dist", - requirements = { - "3.9": "@mediapipe//:requirements_lock.txt", - "3.10": "@mediapipe//:requirements_lock_3_10.txt", - "3.11": "@mediapipe//:requirements_lock_3_11.txt", - "3.12": "@mediapipe//:requirements_lock_3_12.txt", - }, - #local_wheel_inclusion_list = ["mediapipe*"], - #local_wheel_workspaces = ["//:WORKSPACE"], -) - -load("@org_tensorflow//third_party/xla/third_party/py:python_init_toolchains.bzl", "python_init_toolchains") -python_init_toolchains() - -load("@org_tensorflow//third_party/xla/third_party/py:python_init_pip.bzl", "python_init_pip") -python_init_pip() - -load("@pypi//:requirements.bzl", "install_deps") -install_deps() -# End hermetic Python initialization -load("@org_tensorflow//tensorflow:workspace2.bzl", "workspace") -workspace() -load("@org_tensorflow//tensorflow:workspace1.bzl", "workspace") -workspace() -load("@org_tensorflow//tensorflow:workspace0.bzl", "workspace") -workspace() - -# required after update to mp 0.10.18 -load( - "@org_tensorflow//third_party/gpus/cuda/hermetic:cuda_configure.bzl", - "cuda_configure", + sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", ) -cuda_configure(name = "local_config_cuda") - - # Initialize bazel package rules' external dependencies. load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") @@ -529,13 +506,9 @@ gl_cpp_workspace5() load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") grpc_deps() -http_archive( # 1.74.1 - name = "com_github_grpc_grpc", - urls = [ - "https://github.com/grpc/grpc/archive/893bdadd56dbb75fb156175afdaa2b0d47e1c15b.tar.gz", - ], - strip_prefix = "grpc-893bdadd56dbb75fb156175afdaa2b0d47e1c15b", -) + +load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") +grpc_extra_deps() # cxxopts http_archive( @@ -577,7 +550,7 @@ load("@com_github_jupp0r_prometheus_cpp//bazel:repositories.bzl", "prometheus_cp prometheus_cpp_repositories() load("@rules_foreign_cc//foreign_cc:cmake.bzl", "cmake") -load("@mediapipe//third_party/model_api:model_api.bzl", "workspace_model_api") +load("@ovms//third_party/model_api:model_api.bzl", "workspace_model_api") workspace_model_api() new_local_repository( @@ -586,6 +559,15 @@ new_local_repository( path = "third_party/mediapipe_calculators", ) +# Eigen — referenced by mediapipe upstream calculators (e.g. matrix_to_vector). +http_archive( + name = "eigen", + build_file = "@mediapipe//third_party:eigen.BUILD", + sha256 = "35c6126e246585d9cf6600b65471582c2701aae64b784a6fd19168a90cfc841e", + strip_prefix = "eigen-ea13a98decd497a8c5588fb5de71b57bcf10d864", + urls = ["https://gitlab.com/libeigen/eigen/-/archive/ea13a98decd497a8c5588fb5de71b57bcf10d864/eigen-ea13a98decd497a8c5588fb5de71b57bcf10d864.tar.gz"], +) + http_archive( name = "nlohmann_json", sha256 = "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406", diff --git a/ci/lib_search.py b/ci/lib_search.py index af07283335..cd583ea6c0 100644 --- a/ci/lib_search.py +++ b/ci/lib_search.py @@ -122,8 +122,6 @@ def check_dir(start_dir): 'upb_platform_fix.patch', 'upb_warning_turn_off.patch', 'partial_2.18.patch', - 'tf_2.18_logging.patch', - 'tf_nsync_chrono.patch', 'vehicle_images.txt', 'bazel_rules_apple.patch', "go.sum", @@ -237,8 +235,6 @@ def check_func(start_dir): 'tf_graph_info_multilinecomment.patch', 'tftext.patch', 'partial_2.18.patch', - 'tf_2.18_logging.patch', - 'tf_nsync_chrono.patch', 'zlib.LICENSE.txt', 'bazel_rules_apple.patch', 'yarn.lock', diff --git a/common_settings.bzl b/common_settings.bzl index 5785c3b60b..f02c06f988 100644 --- a/common_settings.bzl +++ b/common_settings.bzl @@ -144,8 +144,11 @@ LINUX_COMMON_STATIC_LIBS_COPTS_WITHOUT_VISIBILITY = [ "-Wno-sign-compare", "-Werror", # ov::Tensor::data method call results in deprecated warning and we use it in multiple places + # Some upstream dependencies (e.g. Abseil / Google Cloud C++) still use deprecated APIs + # that are not actionable for OVMS and must not fail the build under -Werror. "-Wno-deprecated-declarations", "-Werror", + "-Wno-error=deprecated-declarations", "-Wimplicit-fallthrough", "-fcf-protection=full", "-Wformat", @@ -208,6 +211,8 @@ COMMON_STATIC_TEST_COPTS = select({ "-Wall", "-Wno-unknown-pragmas", "-Werror", + "-Wno-deprecated-declarations", + "-Wno-error=deprecated-declarations", "-Isrc", "-fconcepts", # for gmock related utils "-fvisibility=hidden",# Needed for pybind targets diff --git a/external/tf_2.18_logging.patch b/external/tf_2.18_logging.patch deleted file mode 100644 index e8108379f9..0000000000 --- a/external/tf_2.18_logging.patch +++ /dev/null @@ -1,315 +0,0 @@ -diff --git a/third_party/xla/third_party/tsl/tsl/platform/default/BUILD b/third_party/xla/third_party/tsl/tsl/platform/default/BUILD -index 785a44cfff0..d98afb28d11 100644 ---- a/third_party/xla/third_party/tsl/tsl/platform/default/BUILD -+++ b/third_party/xla/third_party/tsl/tsl/platform/default/BUILD -@@ -262,6 +262,18 @@ cc_library( - ], - ) - -+cc_library( -+ name = "glog", -+ #hdrs = ["@com_github_glog_glog//:glog/logging.h",] -+ visibility = ["//visibility:public"], -+ deps = ["@com_github_glog_glog//:glog",], -+) -+ -+cc_library( -+ name = "log_macros", -+ hdrs = ["log_macros.h"], -+) -+ - cc_library( - name = "logging", - srcs = ["logging.cc"], -@@ -281,6 +293,8 @@ cc_library( - "@com_google_absl//absl/base", - "@com_google_absl//absl/base:log_severity", - "@com_google_absl//absl/strings", -+ ":glog", -+ ":log_macros", - ], - ) - -diff --git a/third_party/xla/third_party/tsl/tsl/platform/default/log_macros.h b/third_party/xla/third_party/tsl/tsl/platform/default/log_macros.h -new file mode 100644 -index 00000000000..221d42c9e77 ---- /dev/null -+++ b/third_party/xla/third_party/tsl/tsl/platform/default/log_macros.h -@@ -0,0 +1,102 @@ -+#pragma once -+#define LOG(severity) _TF_LOG_##severity -+ -+// An instance of `LOG_EVERY_N` increments a hidden zero-initialized counter -+// every time execution passes through it and logs the specified message when -+// the counter's value is a multiple of `n`, doing nothing otherwise. Each -+// instance has its own counter. The counter's value can be logged by streaming -+// the symbol `COUNTER`. `LOG_EVERY_N` is thread-safe. -+// Example: -+// -+// for (const auto& user : all_users) { -+// LOG_EVERY_N(INFO, 1000) << "Processing user #" << COUNTER; -+// ProcessUser(user); -+// } -+ -+// CHECK dies with a fatal error if condition is not true. It is *not* -+// controlled by NDEBUG, so the check will be executed regardless of -+// compilation mode. Therefore, it is safe to do things like: -+// CHECK(fp->Write(x) == 4) -+#define CHECK(condition) \ -+ if (TF_PREDICT_FALSE(!(condition))) \ -+ LOG(FATAL) << "Check failed: " #condition " " -+// `LOG_FIRST_N` behaves like `LOG_EVERY_N` except that the specified message is -+// logged when the counter's value is less than `n`. `LOG_FIRST_N` is -+// thread-safe. -+ -+#define VLOG(level) \ -+ TF_PREDICT_TRUE(!VLOG_IS_ON(level)) \ -+ ? (void)0 \ -+ : ::tsl::internal::Voidifier() & \ -+ ::tsl::internal::LogMessage(__FILE__, __LINE__, \ -+ absl::LogSeverity::kInfo) -+ -+// `DVLOG` behaves like `VLOG` in debug mode (i.e. `#ifndef NDEBUG`). -+// Otherwise, it compiles away and does nothing. -+#ifndef NDEBUG -+#ifndef DVLOG -+#define DVLOG VLOG -+#endif -+#else -+#ifndef DVLOG -+#define DVLOG(verbose_level) \ -+ while (false && (verbose_level) > 0) ::tsl::internal::LogMessageNull() -+#endif -+#endif -+ -+ -+// In optimized mode, use CheckOpString to hint to compiler that -+// the while condition is unlikely. -+#ifndef CHECK_OP_LOG -+#define CHECK_OP_LOG(name, op, val1, val2) \ -+ while (::tsl::internal::CheckOpString _result{ \ -+ ::tsl::internal::name##Impl( \ -+ ::tsl::internal::GetReferenceableValue(val1), \ -+ ::tsl::internal::GetReferenceableValue(val2), \ -+ #val1 " " #op " " #val2)}) \ -+ ::tsl::internal::LogMessageFatal(__FILE__, __LINE__) << *(_result.str_) -+#endif -+#ifndef CHECK_OP -+#define CHECK_OP(name, op, val1, val2) CHECK_OP_LOG(name, op, val1, val2) -+#endif -+ -+// CHECK_EQ/NE/... -+#define CHECK_EQ(val1, val2) CHECK_OP(_EQ, ==, val1, val2) -+#define CHECK_NE(val1, val2) CHECK_OP(_NE, !=, val1, val2) -+#define CHECK_LE(val1, val2) CHECK_OP(_LE, <=, val1, val2) -+#define CHECK_LT(val1, val2) CHECK_OP(_LT, < , val1, val2) -+#define CHECK_GE(val1, val2) CHECK_OP(_GE, >=, val1, val2) -+#define CHECK_GT(val1, val2) CHECK_OP(_GT, > , val1, val2) -+ -+ -+#ifndef NDEBUG -+// DCHECK_EQ/NE/... -+#define DCHECK(condition) CHECK(condition) -+#define DCHECK_EQ(val1, val2) CHECK_EQ(val1, val2) -+#define DCHECK_NE(val1, val2) CHECK_NE(val1, val2) -+#define DCHECK_LE(val1, val2) CHECK_LE(val1, val2) -+#define DCHECK_LT(val1, val2) CHECK_LT(val1, val2) -+#define DCHECK_GE(val1, val2) CHECK_GE(val1, val2) -+#define DCHECK_GT(val1, val2) CHECK_GT(val1, val2) -+ -+#else -+ -+#define DCHECK(condition) \ -+ while (false && (condition)) LOG(FATAL) -+ -+// NDEBUG is defined, so DCHECK_EQ(x, y) and so on do nothing. -+// However, we still want the compiler to parse x and y, because -+// we don't want to lose potentially useful errors and warnings. -+// _DCHECK_NOP is a helper, and should not be used outside of this file. -+#define _TF_DCHECK_NOP(x, y) \ -+ while (false && ((void)(x), (void)(y), 0)) LOG(FATAL) -+ -+#define DCHECK_EQ(x, y) _TF_DCHECK_NOP(x, y) -+#define DCHECK_NE(x, y) _TF_DCHECK_NOP(x, y) -+#define DCHECK_LE(x, y) _TF_DCHECK_NOP(x, y) -+#define DCHECK_LT(x, y) _TF_DCHECK_NOP(x, y) -+#define DCHECK_GE(x, y) _TF_DCHECK_NOP(x, y) -+#define DCHECK_GT(x, y) _TF_DCHECK_NOP(x, y) -+ -+#endif -+ -diff --git a/third_party/xla/third_party/tsl/tsl/platform/default/logging.h b/third_party/xla/third_party/tsl/tsl/platform/default/logging.h -index bc94f79c060..bc6a2eb858c 100644 ---- a/third_party/xla/third_party/tsl/tsl/platform/default/logging.h -+++ b/third_party/xla/third_party/tsl/tsl/platform/default/logging.h -@@ -44,6 +44,7 @@ limitations under the License. - // which already defined them itself. Presumably all Google libraries will - // support the same syntax for these so it should not be a big deal if they - // end up using our definitions instead. -+#define COMPACT_GOOGLE_LOG_QFATAL COMPACT_GOOGLE_LOG_ERROR - #undef LOG - #undef LOG_EVERY_N - #undef LOG_FIRST_N -@@ -77,6 +78,17 @@ limitations under the License. - - #undef PCHECK - -+ -+#pragma GCC diagnostic push -+#pragma GCC diagnostic ignored "-Wsign-compare" -+#include "glog/logging.h" -+#pragma GCC diagnostic pop -+ -+#ifndef LOG -+#include "log_macros.h" -+#endif -+ -+ - namespace tsl { - - namespace internal { -@@ -157,43 +169,6 @@ class LogMessageNull : public std::basic_ostringstream { - #define _TF_LOG_DFATAL _TF_LOG_FATAL - #endif - --#define LOG(severity) _TF_LOG_##severity -- --#ifdef IS_MOBILE_PLATFORM -- --// Turn VLOG off when under mobile devices for considerations of binary size. --#define VLOG_IS_ON(lvl) ((lvl) <= 0) -- --#else -- --// Otherwise, set TF_CPP_MAX_VLOG_LEVEL environment to update minimum log level --// of VLOG, or TF_CPP_VMODULE to set the minimum log level for individual --// translation units. --#define VLOG_IS_ON(lvl) \ -- (([](int level, const char* fname) { \ -- static const bool vmodule_activated = \ -- ::tsl::internal::LogMessage::VmoduleActivated(fname, level); \ -- return vmodule_activated; \ -- })(lvl, __FILE__)) -- --#endif -- --#define VLOG(level) \ -- TF_PREDICT_TRUE(!VLOG_IS_ON(level)) \ -- ? (void)0 \ -- : ::tsl::internal::Voidifier() & \ -- ::tsl::internal::LogMessage(__FILE__, __LINE__, \ -- absl::LogSeverity::kInfo) -- --// `DVLOG` behaves like `VLOG` in debug mode (i.e. `#ifndef NDEBUG`). --// Otherwise, it compiles away and does nothing. --#ifndef NDEBUG --#define DVLOG VLOG --#else --#define DVLOG(verbose_level) \ -- while (false && (verbose_level) > 0) ::tsl::internal::LogMessageNull() --#endif -- - class LogEveryNState { - public: - bool ShouldLog(int n); -@@ -260,26 +235,6 @@ class LogEveryNSecState { - logging_internal_stateful_condition_do_log; \ - logging_internal_stateful_condition_do_log = false) - --// An instance of `LOG_EVERY_N` increments a hidden zero-initialized counter --// every time execution passes through it and logs the specified message when --// the counter's value is a multiple of `n`, doing nothing otherwise. Each --// instance has its own counter. The counter's value can be logged by streaming --// the symbol `COUNTER`. `LOG_EVERY_N` is thread-safe. --// Example: --// --// for (const auto& user : all_users) { --// LOG_EVERY_N(INFO, 1000) << "Processing user #" << COUNTER; --// ProcessUser(user); --// } --#define LOG_EVERY_N(severity, n) \ -- LOGGING_INTERNAL_STATEFUL_CONDITION(EveryN, true, n) \ -- LOG(severity) --// `LOG_FIRST_N` behaves like `LOG_EVERY_N` except that the specified message is --// logged when the counter's value is less than `n`. `LOG_FIRST_N` is --// thread-safe. --#define LOG_FIRST_N(severity, n) \ -- LOGGING_INTERNAL_STATEFUL_CONDITION(FirstN, true, n) \ -- LOG(severity) - // `LOG_EVERY_POW_2` behaves like `LOG_EVERY_N` except that the specified - // message is logged when the counter's value is a power of 2. - // `LOG_EVERY_POW_2` is thread-safe. -@@ -297,14 +252,6 @@ class LogEveryNSecState { - LOGGING_INTERNAL_STATEFUL_CONDITION(EveryNSec, true, n_seconds) \ - LOG(severity) - --// CHECK dies with a fatal error if condition is not true. It is *not* --// controlled by NDEBUG, so the check will be executed regardless of --// compilation mode. Therefore, it is safe to do things like: --// CHECK(fp->Write(x) == 4) --#define CHECK(condition) \ -- if (TF_PREDICT_FALSE(!(condition))) \ -- LOG(FATAL) << "Check failed: " #condition " " -- - // Function is overloaded for integral types to allow static const - // integrals declared in classes and not defined to be used as arguments to - // CHECK* macros. It's not encouraged though. -@@ -481,58 +428,6 @@ inline string* Check_GTImpl(const T1& v1, const T2& v2, const char* exprtext) { - - #undef TF_DEFINE_CHECK_OP_IMPL - --// In optimized mode, use CheckOpString to hint to compiler that --// the while condition is unlikely. --#define CHECK_OP_LOG(name, op, val1, val2) \ -- while (::tsl::internal::CheckOpString _result{::tsl::internal::name##Impl( \ -- ::tsl::internal::GetReferenceableValue(val1), \ -- ::tsl::internal::GetReferenceableValue(val2), #val1 " " #op " " #val2)}) \ -- ::tsl::internal::LogMessageFatal(__FILE__, __LINE__) << *(_result.str_) -- --#define CHECK_OP(name, op, val1, val2) CHECK_OP_LOG(name, op, val1, val2) -- --// CHECK_EQ/NE/... --#define CHECK_EQ(val1, val2) CHECK_OP(Check_EQ, ==, val1, val2) --#define CHECK_NE(val1, val2) CHECK_OP(Check_NE, !=, val1, val2) --#define CHECK_LE(val1, val2) CHECK_OP(Check_LE, <=, val1, val2) --#define CHECK_LT(val1, val2) CHECK_OP(Check_LT, <, val1, val2) --#define CHECK_GE(val1, val2) CHECK_OP(Check_GE, >=, val1, val2) --#define CHECK_GT(val1, val2) CHECK_OP(Check_GT, >, val1, val2) --#define CHECK_NOTNULL(val) \ -- ::tsl::internal::CheckNotNull(__FILE__, __LINE__, \ -- "'" #val "' Must be non NULL", (val)) -- --#ifndef NDEBUG --// DCHECK_EQ/NE/... --#define DCHECK(condition) CHECK(condition) --#define DCHECK_EQ(val1, val2) CHECK_EQ(val1, val2) --#define DCHECK_NE(val1, val2) CHECK_NE(val1, val2) --#define DCHECK_LE(val1, val2) CHECK_LE(val1, val2) --#define DCHECK_LT(val1, val2) CHECK_LT(val1, val2) --#define DCHECK_GE(val1, val2) CHECK_GE(val1, val2) --#define DCHECK_GT(val1, val2) CHECK_GT(val1, val2) -- --#else -- --#define DCHECK(condition) \ -- while (false && (condition)) LOG(FATAL) -- --// NDEBUG is defined, so DCHECK_EQ(x, y) and so on do nothing. --// However, we still want the compiler to parse x and y, because --// we don't want to lose potentially useful errors and warnings. --// _DCHECK_NOP is a helper, and should not be used outside of this file. --#define _TF_DCHECK_NOP(x, y) \ -- while (false && ((void)(x), (void)(y), 0)) LOG(FATAL) -- --#define DCHECK_EQ(x, y) _TF_DCHECK_NOP(x, y) --#define DCHECK_NE(x, y) _TF_DCHECK_NOP(x, y) --#define DCHECK_LE(x, y) _TF_DCHECK_NOP(x, y) --#define DCHECK_LT(x, y) _TF_DCHECK_NOP(x, y) --#define DCHECK_GE(x, y) _TF_DCHECK_NOP(x, y) --#define DCHECK_GT(x, y) _TF_DCHECK_NOP(x, y) -- --#endif -- - // These are for when you don't want a CHECK failure to print a verbose - // stack trace. The implementation of CHECK* in this file already doesn't. - #define QCHECK(condition) CHECK(condition) diff --git a/external/tf_nsync_chrono.patch b/external/tf_nsync_chrono.patch deleted file mode 100644 index 43671fc5e6..0000000000 --- a/external/tf_nsync_chrono.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl -index be83c971749..bd80e19896e 100644 ---- a/tensorflow/workspace2.bzl -+++ b/tensorflow/workspace2.bzl -@@ -396,7 +396,7 @@ def _tf_repositories(): - - tf_http_archive( - name = "nsync", -- patch_file = ["//third_party:nsync.patch"], -+ patch_file = ["//third_party:nsync.patch", "//third_party:nsync_chrono.patch"], - sha256 = "1d63e967973733d2c97e841e3c05fac4d3fa299f01d14c86f2695594c7a4a2ec", - strip_prefix = "nsync-1.29.2", - system_build_file = "//third_party/systemlibs:nsync.BUILD", -diff --git a/third_party/nsync_chrono.patch b/third_party/nsync_chrono.patch -new file mode 100644 -index 00000000000..7f0f8539727 ---- /dev/null -+++ b/third_party/nsync_chrono.patch -@@ -0,0 +1,12 @@ -+diff --git a/platform/c++11/platform.h b/platform/c++11/platform.h -+index 2c80e0b..4fbc6d0 100644 -+--- a/platform/c++11/platform.h -++++ b/platform/c++11/platform.h -+@@ -28,6 +28,7 @@ -+ -+ #include -+ #include -++#include -+ -+ #include "nsync_cpp.h" -+ diff --git a/src/audio/speech_to_text/s2t_servable.hpp b/src/audio/speech_to_text/s2t_servable.hpp index 237365cd20..2ceb7b1136 100644 --- a/src/audio/speech_to_text/s2t_servable.hpp +++ b/src/audio/speech_to_text/s2t_servable.hpp @@ -24,13 +24,10 @@ #include #include +#include "absl/status/status.h" #include "src/audio/speech_to_text/s2t_executor.hpp" #include "src/status.hpp" -namespace absl { -class Status; -} // namespace absl - namespace mediapipe { class S2tCalculatorOptions; } // namespace mediapipe diff --git a/src/dags/dl_node.cpp b/src/dags/dl_node.cpp index 65fafe09e0..bdfdeb64ee 100644 --- a/src/dags/dl_node.cpp +++ b/src/dags/dl_node.cpp @@ -132,6 +132,7 @@ Status DLNode::fetchResults(TensorWithSourceMap& outputs, ov::InferRequest& infe if (!status.ok()) { SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Could not clone result tensor; node: {}; session: {}; model name: {}; output: {}", getName(), + sessionKey, this->modelName, realModelOutputName); return status; diff --git a/src/embeddings/embeddings_api.cpp b/src/embeddings/embeddings_api.cpp index 042545952a..a38d3e8e0c 100644 --- a/src/embeddings/embeddings_api.cpp +++ b/src/embeddings/embeddings_api.cpp @@ -148,8 +148,7 @@ absl::Status EmbeddingsHandler::parseResponse(StringBuffer& buffer, const ov::Te writer.String("embedding"); if (getEncodingFormat() == EmbeddingsRequest::EncodingFormat::BASE64) { std::string_view sv2(reinterpret_cast(batch_result.data()), batch_result.size() * sizeof(float)); - std::string escaped; - absl::Base64Escape(sv2, &escaped); + const std::string escaped = absl::Base64Escape(sv2); writer.String(escaped.c_str()); } else { writer.StartArray(); diff --git a/src/filesystem/azurestorage.cpp b/src/filesystem/azurestorage.cpp index da4bee8174..c353efc7dd 100644 --- a/src/filesystem/azurestorage.cpp +++ b/src/filesystem/azurestorage.cpp @@ -24,7 +24,7 @@ namespace ovms { -const std::string UNAVAILABLE_PATH_ERROR = "Unable to access path: {}"; +inline constexpr std::string_view UNAVAILABLE_PATH_ERROR = "Unable to access path: {}"; const std::string AzureStorageAdapter::extractAzureStorageExceptionMessage(const Azure::Storage::StorageException& e) { if (!e.Message.empty()) { diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 9a602f8c8a..f3d7e371a3 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -33,6 +33,9 @@ #ifndef _WIN32 #include #endif + +#include "absl/status/status.h" + #include "src/port/rapidjson_stringbuffer.hpp" #include "src/port/rapidjson_writer.hpp" @@ -232,7 +235,7 @@ Status HttpRestApiHandler::processServerMetadataKFSRequest(const HttpRequestComp } std::string output; google::protobuf::util::JsonPrintOptions opts; - google::protobuf::util::Status status = google::protobuf::util::MessageToJsonString(grpc_response, &output, opts); + absl::Status status = google::protobuf::util::MessageToJsonString(grpc_response, &output, opts); if (!status.ok()) { return StatusCode::INTERNAL_ERROR; } @@ -898,9 +901,9 @@ Status HttpRestApiHandler::processModelMetadataKFSRequest(const HttpRequestCompo } std::string output; google::protobuf::util::JsonPrintOptions opts; - // This parameter forces JSON writer to not omit empty shape in case of scalar tensor - opts.always_print_primitive_fields = true; - google::protobuf::util::Status status = google::protobuf::util::MessageToJsonString(grpc_response, &output, opts); + // Renamed from always_print_primitive_fields in protobuf 6.30. + opts.always_print_fields_with_no_presence = true; + absl::Status status = google::protobuf::util::MessageToJsonString(grpc_response, &output, opts); if (!status.ok()) { return StatusCode::JSON_SERIALIZATION_ERROR; } diff --git a/src/image_gen/imagegenutils.cpp b/src/image_gen/imagegenutils.cpp index 7bedc37f14..5f7b4d63b1 100644 --- a/src/image_gen/imagegenutils.cpp +++ b/src/image_gen/imagegenutils.cpp @@ -594,7 +594,7 @@ std::variant> generateJSONResponseFro auto& imagesAsStrings = std::get>(imagesAsStringsOrStatus); std::vector base64images(imagesAsStrings.size()); for (size_t i = 0; i < imagesAsStrings.size(); ++i) { - absl::Base64Escape(imagesAsStrings[i], &base64images[i]); + base64images[i] = absl::Base64Escape(imagesAsStrings[i]); } auto output = generateJSONResponseFromB64Images(base64images); return output; diff --git a/src/kfs_frontend/kfs_graph_executor_impl.cpp b/src/kfs_frontend/kfs_graph_executor_impl.cpp index f29272b3dc..44ebda4e68 100644 --- a/src/kfs_frontend/kfs_graph_executor_impl.cpp +++ b/src/kfs_frontend/kfs_graph_executor_impl.cpp @@ -111,9 +111,9 @@ Status MediapipeGraphExecutor::infer(const KFSRequest* request, KFSResponse* res Status MediapipeGraphExecutor::inferStream( const KFSRequest& firstRequest, - grpc_impl::ServerReaderWriterInterface& serverReaderWriter, + grpc::ServerReaderWriterInterface& serverReaderWriter, const ExecutionContext& executionContext) { - return this->inferStreamTyped>(firstRequest, serverReaderWriter, executionContext); + return this->inferStreamTyped>(firstRequest, serverReaderWriter, executionContext); } // Utilities @@ -513,7 +513,7 @@ Status receiveAndSerializePacket(const ::mediapipe::Packe HANDLE_PACKET_RECEIVAL_EXCEPTIONS(); } -static Status getRequestInput(google::protobuf::internal::RepeatedPtrIterator& itr, const std::string& requestedName, const KFSRequest& request) { +static Status getRequestInput(KFSInputTensorIteratorType& itr, const std::string& requestedName, const KFSRequest& request) { auto requestInputItr = std::find_if(request.inputs().begin(), request.inputs().end(), [&requestedName](const ::KFSRequest::InferInputTensor& tensor) { return tensor.name() == requestedName; }); if (requestInputItr == request.inputs().end()) { std::stringstream ss; diff --git a/src/kfs_frontend/kfs_utils.hpp b/src/kfs_frontend/kfs_utils.hpp index 40592d5f30..a25ad43860 100644 --- a/src/kfs_frontend/kfs_utils.hpp +++ b/src/kfs_frontend/kfs_utils.hpp @@ -38,8 +38,8 @@ using KFSShapeType = google::protobuf::RepeatedField; using KFSGetModelStatusRequest = inference::ModelReadyRequest; using KFSGetModelStatusResponse = inference::ModelReadyResponse; using KFSDataType = std::string; -using KFSInputTensorIteratorType = google::protobuf::internal::RepeatedPtrIterator; -using KFSOutputTensorIteratorType = google::protobuf::internal::RepeatedPtrIterator; +using KFSInputTensorIteratorType = google::protobuf::RepeatedPtrField<::inference::ModelInferRequest_InferInputTensor>::const_iterator; +using KFSOutputTensorIteratorType = google::protobuf::RepeatedPtrField<::inference::ModelInferResponse_InferOutputTensor>::const_iterator; namespace ovms { class Status; diff --git a/src/kfserving_api/BUILD b/src/kfserving_api/BUILD index 7991db132f..40c8229924 100644 --- a/src/kfserving_api/BUILD +++ b/src/kfserving_api/BUILD @@ -13,18 +13,36 @@ # See the License for the specific language governing permissions and # limitations under the License. # -load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") + +proto_library( + name = "kfserving_api_proto", + srcs = ["grpc_predict_v2.proto"], + visibility = ["//visibility:public"], +) cc_proto_library( + name = "kfserving_api_cpp_proto", + deps = [":kfserving_api_proto"], + visibility = ["//visibility:public"], +) + +cc_grpc_library( + name = "kfserving_api_cpp_grpc", + srcs = [":kfserving_api_proto"], + grpc_only = True, + deps = [":kfserving_api_cpp_proto"], + visibility = ["//visibility:public"], +) + +# Aggregate target that both the messages and the grpc service, keeping the +# historical `kfserving_api_cpp` label used across the codebase. +cc_library( name = "kfserving_api_cpp", - srcs = ["grpc_predict_v2.proto",], - deps = [], - cc_libs = ["@com_google_protobuf//:protobuf"], - protoc = "@com_google_protobuf//:protoc", - default_runtime = "@com_google_protobuf//:protobuf", - use_grpc_plugin = True, # ?? - testonly = False, # ?? - visibility = [ - "//visibility:public", + deps = [ + ":kfserving_api_cpp_grpc", + ":kfserving_api_cpp_proto", ], + visibility = ["//visibility:public"], ) diff --git a/src/mediapipe_internal/mediapipe_graph_executor_interface.hpp b/src/mediapipe_internal/mediapipe_graph_executor_interface.hpp index 5cc2fd658d..646e27f4a3 100644 --- a/src/mediapipe_internal/mediapipe_graph_executor_interface.hpp +++ b/src/mediapipe_internal/mediapipe_graph_executor_interface.hpp @@ -20,7 +20,7 @@ #include "src/execution_context.hpp" #include "src/status.hpp" -namespace grpc_impl { +namespace grpc { template class ServerReaderWriterInterface; } @@ -44,7 +44,7 @@ class MediapipeGraphExecutorInterface { inference::ModelInferResponse* response, const ExecutionContext& executionContext) = 0; virtual Status inferStream(const inference::ModelInferRequest& firstRequest, - grpc_impl::ServerReaderWriterInterface& serverReaderWriter, + grpc::ServerReaderWriterInterface& serverReaderWriter, const ExecutionContext& executionContext) = 0; virtual Status infer(const HttpPayload* request, std::string* response, diff --git a/src/mediapipe_internal/mediapipegraphexecutor.hpp b/src/mediapipe_internal/mediapipegraphexecutor.hpp index 13cd6dceb6..46c8862e45 100644 --- a/src/mediapipe_internal/mediapipegraphexecutor.hpp +++ b/src/mediapipe_internal/mediapipegraphexecutor.hpp @@ -234,7 +234,7 @@ class MediapipeGraphExecutor : public MediapipeGraphExecutorInterface { inference::ModelInferResponse* response, const ExecutionContext& executionContext) override; Status inferStream(const inference::ModelInferRequest& firstRequest, - grpc_impl::ServerReaderWriterInterface& serverReaderWriter, + grpc::ServerReaderWriterInterface& serverReaderWriter, const ExecutionContext& executionContext) override; Status infer(const HttpPayload* request, std::string* response, diff --git a/src/test/deserialization_tests.cpp b/src/test/deserialization_tests.cpp index f59c2a4e91..844392fa95 100644 --- a/src/test/deserialization_tests.cpp +++ b/src/test/deserialization_tests.cpp @@ -109,7 +109,7 @@ const std::string CAPIPredictRequest::DATA{1 * sizeof(float) * DUMMY_MODEL_INPUT // dispose connection to CAPI Predict TEST_F(CAPIPredictRequest, ShouldSuccessForSupportedPrecision) { ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); ov::InferRequest inferRequest = compiledModel.create_infer_request(); InputSink inputSink(inferRequest); @@ -329,7 +329,7 @@ class KserveGRPCPredictRequest : public KserveGRPCPredict { TEST_F(KserveGRPCPredictRequest, ShouldSuccessForSupportedPrecision) { ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); ov::InferRequest inferRequest = compiledModel.create_infer_request(); InputSink inputSink(inferRequest); diff --git a/src/test/llm/input_processing/audio_decoding_processor_test.cpp b/src/test/llm/input_processing/audio_decoding_processor_test.cpp index badca016c3..03c03dc8c4 100644 --- a/src/test/llm/input_processing/audio_decoding_processor_test.cpp +++ b/src/test/llm/input_processing/audio_decoding_processor_test.cpp @@ -43,9 +43,7 @@ static InputRequest makeChatRequest(ov::genai::ChatHistory chatHistory) { // Encodes binary data to base64. static std::string toBase64(const std::string& raw) { - std::string encoded; - absl::Base64Escape(raw, &encoded); - return encoded; + return absl::Base64Escape(raw); } // Builds a ChatHistory message with a content array containing an input_audio part. diff --git a/src/test/llm/input_processing/input_processing_integration_test.cpp b/src/test/llm/input_processing/input_processing_integration_test.cpp index ab4d6172f3..2a4a6d1772 100644 --- a/src/test/llm/input_processing/input_processing_integration_test.cpp +++ b/src/test/llm/input_processing/input_processing_integration_test.cpp @@ -445,9 +445,7 @@ INSTANTIATE_TEST_SUITE_P(BothEndpoints, InputProcessingIntegrationTest, // --------------------------------------------------------------------------- static std::string toBase64(const std::string& raw) { - std::string encoded; - absl::Base64Escape(raw, &encoded); - return encoded; + return absl::Base64Escape(raw); } TEST_P(InputProcessingIntegrationTest, TextImageAudio_AllModalitiesDecoded) { diff --git a/src/test/ov_utils_test.cpp b/src/test/ov_utils_test.cpp index 085156f393..fd2e09a165 100644 --- a/src/test/ov_utils_test.cpp +++ b/src/test/ov_utils_test.cpp @@ -197,7 +197,7 @@ TEST(OVUtils, GetLayoutFromRTMap) { TEST(OVUtils, ValidatePluginConfigurationPositive) { ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ovms::ModelConfig config; config.setTargetDevice("CPU"); config.setPluginConfig({{"NUM_STREAMS", "10"}}); @@ -208,7 +208,7 @@ TEST(OVUtils, ValidatePluginConfigurationPositive) { TEST(OVUtils, ValidatePluginConfigurationPositiveBatch) { ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ovms::ModelConfig config; config.setTargetDevice("BATCH:CPU(4)"); config.setPluginConfig({{"AUTO_BATCH_TIMEOUT", 10}}); @@ -219,7 +219,7 @@ TEST(OVUtils, ValidatePluginConfigurationPositiveBatch) { TEST(OVUtils, ValidatePluginConfigurationNegative) { ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ovms::ModelConfig config; config.setTargetDevice("CPU"); config.setPluginConfig({{"WRONG_KEY ", "10"}}); @@ -237,7 +237,7 @@ TEST(OVUtils, ValidatePluginConfigurationAllowEnableMmap) { ovms::plugin_config_t pluginConfig = ovms::ModelInstance::prepareDefaultPluginConfig(config); auto status = ovms::validatePluginConfiguration(pluginConfig, "CPU", ieCore); EXPECT_TRUE(status.ok()); - auto model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml", {}, pluginConfig); + auto model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml", {}, pluginConfig); auto compiledModel = ieCore.compile_model(model, "CPU", pluginConfig); } diff --git a/src/test/ovinferrequestqueue_test.cpp b/src/test/ovinferrequestqueue_test.cpp index 8636a77114..86dafef42c 100644 --- a/src/test/ovinferrequestqueue_test.cpp +++ b/src/test/ovinferrequestqueue_test.cpp @@ -30,7 +30,7 @@ using namespace testing; -const std::string DUMMY_MODEL_PATH = std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"; +const std::string DUMMY_MODEL_PATH = std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"; TEST(OVInferRequestQueue, ShortQueue) { ov::Core ieCore; diff --git a/src/test/serialization_tests.cpp b/src/test/serialization_tests.cpp index ce7654502c..fe6e7d62b2 100644 --- a/src/test/serialization_tests.cpp +++ b/src/test/serialization_tests.cpp @@ -357,7 +357,7 @@ TEST_P(SerializeKFSInferOutputTensorNegative, SerializeTensorProtoShouldFailForP TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { KFSResponse response; ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); ov::InferRequest inferRequest = compiledModel.create_infer_request(); ovms::tensor_map_t tenMap; @@ -382,7 +382,7 @@ TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecisionWithuseSharedOutputContent) { KFSResponse response; ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); ov::InferRequest inferRequest = compiledModel.create_infer_request(); ovms::tensor_map_t tenMap; @@ -408,7 +408,7 @@ TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecisionWithuseS TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecisionWithsharedInputContentsNotUsed) { KFSResponse response; ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); ov::InferRequest inferRequest = compiledModel.create_infer_request(); ovms::tensor_map_t tenMap; @@ -464,7 +464,7 @@ class CAPISerialization : public ::testing::TestWithParam { TEST(SerializeCAPITensorSingle, NegativeMismatchBetweenTensorInfoAndTensorPrecision) { InferenceResponse response{"dummy", 1}; ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); ov::InferRequest inferRequest = compiledModel.create_infer_request(); ovms::tensor_map_t tenMap; @@ -486,7 +486,7 @@ TEST(SerializeCAPITensorSingle, NegativeMismatchBetweenTensorInfoAndTensorPrecis TEST(SerializeCAPITensorSingle, NegativeMismatchBetweenTensorInfoAndTensorShape) { InferenceResponse response{"dummy", 1}; ov::Core ieCore; - std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().string() + "/src/test/dummy/1/dummy.xml"); ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); ov::InferRequest inferRequest = compiledModel.create_infer_request(); ovms::tensor_map_t tenMap; diff --git a/src/test/test_models.hpp b/src/test/test_models.hpp index c6ff0ee054..b0e5c82e67 100644 --- a/src/test/test_models.hpp +++ b/src/test/test_models.hpp @@ -20,16 +20,16 @@ #include "platform_utils.hpp" #include "src/shape.hpp" -const std::string dummy_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/dummy", false); -const std::string dummy_fp64_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/dummy_fp64", false); -const std::string sum_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/add_two_inputs_model", false); -const std::string increment_1x3x4x5_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/increment_1x3x4x5", false); -const std::string passthrough_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/passthrough", false); -const std::string passthrough_string_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/passthrough_string", false); -const std::string dummy_saved_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/dummy_saved_model", false); -const std::string dummy_tflite_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/dummy_tflite", false); -const std::string scalar_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/scalar", false); -const std::string no_name_output_model_location = getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/no_name_output", false); +const std::string dummy_model_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/dummy", false); +const std::string dummy_fp64_model_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/dummy_fp64", false); +const std::string sum_model_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/add_two_inputs_model", false); +const std::string increment_1x3x4x5_model_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/increment_1x3x4x5", false); +const std::string passthrough_model_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/passthrough", false); +const std::string passthrough_string_model_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/passthrough_string", false); +const std::string dummy_saved_model_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/dummy_saved_model", false); +const std::string dummy_tflite_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/dummy_tflite", false); +const std::string scalar_model_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/scalar", false); +const std::string no_name_output_model_location = getGenericFullPathForSrcTest(std::filesystem::path(std::filesystem::current_path()) / "src/test/no_name_output", false); constexpr const char* DUMMY_MODEL_INPUT_NAME = "b"; constexpr const char* DUMMY_MODEL_OUTPUT_NAME = "a"; diff --git a/src/test/text2image_test.cpp b/src/test/text2image_test.cpp index 49e7c36c3c..9f6eafe4ef 100644 --- a/src/test/text2image_test.cpp +++ b/src/test/text2image_test.cpp @@ -1232,7 +1232,7 @@ TEST(ImageGenCalculatorOptionsTest, PositiveRelativePathToGraphPbtxt) { auto imageGenArgs = std::get(imageGenArgsOrStatus); #ifdef _WIN32 ASSERT_EQ(getGenericFullPathForSrcTest(imageGenArgs.modelsPath), - getGenericFullPathForSrcTest(std::filesystem::current_path().u8string() + "/src/test/dummy\\.\\", false)) + getGenericFullPathForSrcTest(std::filesystem::current_path().string() + "/src/test/dummy\\.\\", false)) << imageGenArgs.modelsPath; #else ASSERT_EQ(imageGenArgs.modelsPath, "/ovms/src/test/dummy/./") diff --git a/third_party/BUILD b/third_party/BUILD index 4fc3e0569a..07b31f20f7 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -59,7 +59,7 @@ alias( name = "curl", actual = select({ "//src:windows": "@windows_curl//:curl", - "//conditions:default": "@linux_curl//:curl", + "//conditions:default": "@curl//:curl", }), visibility = ["//visibility:public"], ) diff --git a/third_party/absl/BUILD b/third_party/absl/BUILD new file mode 100644 index 0000000000..96490186ce --- /dev/null +++ b/third_party/absl/BUILD @@ -0,0 +1,17 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +exports_files(glob(["*.patch"])) diff --git a/third_party/absl/absl_constexpr_fix.patch b/third_party/absl/absl_constexpr_fix.patch new file mode 100644 index 0000000000..8029dd1728 --- /dev/null +++ b/third_party/absl/absl_constexpr_fix.patch @@ -0,0 +1,41 @@ +Work around gcc 13.3 rejecting the constexpr nullptr comparison in +absl::container_internal::hash_policy_traits::get_hash_slot_fn (raw_hash_set.h), +and guard ABSL_RETURN_IF_ERROR / ABSL_ASSIGN_OR_RETURN definitions in +absl/status/status_macros.h so mediapipe's identically-named macros do not +trigger -Werror=macro-redefined when both headers land in the same TU. + +--- a/absl/container/internal/raw_hash_set.h ++++ b/absl/container/internal/raw_hash_set.h +@@ -3860,7 +3860,7 @@ + size_t{(std::numeric_limits::max)()}); + static constexpr size_t kBackingArrayAlignment = + BackingArrayAlignment(alignof(slot_type)); +- static constexpr PolicyFunctions value = { ++ static const PolicyFunctions value = { + static_cast(sizeof(key_type)), + static_cast(sizeof(value_type)), + static_cast(sizeof(slot_type)), +--- a/absl/status/status_macros.h ++++ b/absl/status/status_macros.h +@@ -86,8 +86,10 @@ + // ABSL_RETURN_IF_ERROR(foo.Method(args...)); + // return absl::OkStatus(); + // } ++#ifndef ABSL_RETURN_IF_ERROR + #define ABSL_RETURN_IF_ERROR(expr) \ + ABSL_INTERNAL_STATUS_MACROS_RETURN_IF_ERROR_IMPL_(return, expr) ++#endif + + // Executes an expression `rexpr` that returns an `absl::StatusOr`. On OK, + // moves its value into the variable defined by `lhs`, otherwise returns +@@ -150,8 +152,10 @@ + // Example: Logging the error on failure. + // ABSL_ASSIGN_OR_RETURN(ValueType value, MaybeGetValue(query), _.LogError()); + // ++#ifndef ABSL_ASSIGN_OR_RETURN + #define ABSL_ASSIGN_OR_RETURN(...) \ + ABSL_INTERNAL_STATUS_MACROS_ASSIGN_OR_RETURN_IMPL_(return, __VA_ARGS__) ++#endif + + // ================================================================= + // == Implementation details, do not rely on anything below here. == diff --git a/third_party/grpc/BUILD b/third_party/grpc/BUILD new file mode 100644 index 0000000000..3d5a16acf7 --- /dev/null +++ b/third_party/grpc/BUILD @@ -0,0 +1,17 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +exports_files(glob(["*.patch", "*.diff"])) diff --git a/third_party/grpc/grpc_missing_algorithm_include.patch b/third_party/grpc/grpc_missing_algorithm_include.patch new file mode 100644 index 0000000000..4df5657323 --- /dev/null +++ b/third_party/grpc/grpc_missing_algorithm_include.patch @@ -0,0 +1,23 @@ +diff --git a/src/core/util/glob.cc b/src/core/util/glob.cc +index 1111111..2222222 100644 +--- a/src/core/util/glob.cc ++++ b/src/core/util/glob.cc +@@ -12,6 +12,7 @@ + // See the License for the specific language governing permissions and + // limitations under the License. + ++#include + #include "absl/strings/string_view.h" + + namespace grpc_core { +diff --git a/include/grpc/event_engine/memory_request.h b/include/grpc/event_engine/memory_request.h +--- a/include/grpc/event_engine/memory_request.h ++++ b/include/grpc/event_engine/memory_request.h +@@ -17,6 +17,7 @@ + #include + #include + ++#include + #include "absl/strings/string_view.h" + + namespace grpc_event_engine { diff --git a/third_party/mediapipe/BUILD b/third_party/mediapipe/BUILD new file mode 100644 index 0000000000..0199c874ea --- /dev/null +++ b/third_party/mediapipe/BUILD @@ -0,0 +1,17 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +exports_files(glob(["*.diff", "*.patch"])) diff --git a/third_party/mediapipe/ovms_strip.diff b/third_party/mediapipe/ovms_strip.diff new file mode 100644 index 0000000000..67b91c6009 --- /dev/null +++ b/third_party/mediapipe/ovms_strip.diff @@ -0,0 +1,250 @@ +--- a/mediapipe/framework/tool/mediapipe_proto.bzl ++++ b/mediapipe/framework/tool/mediapipe_proto.bzl +@@ -7,7 +7,6 @@ + load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") + load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") + load("@rules_proto//proto:defs.bzl", _proto_library = "proto_library") +-load("@rules_proto_grpc_js//:defs.bzl", "js_proto_library") + + java_proto_library = native.java_proto_library + java_lite_proto_library = native.java_lite_proto_library +@@ -440,37 +439,8 @@ + visibility = None, + testonly = 0, + compatible_with = None): +- """Generate js_proto_library for mediapipe open source version. +- +- Args: +- name: the name of the js_proto_library. +- srcs: the .proto files of the js_proto_library for Bazel use. +- deps: a list of dependency labels for bazel use ; must be proto_library. +- lib_proto_deps: a list of "_proto" dependency labels. +- visibility: Visibility of this target. +- testonly: test only proto or not. +- compatible_with: a list of environments the rule is compatible with. +- """ +- _ignore = [deps, testonly, compatible_with] +- +- js_deps = replace_deps(lib_proto_deps, "_proto", "_jspb_proto", False) +- js_deps = [d for d in js_deps if not d.endswith("cc_wkt_protos")] +- _proto_library( +- name = replace_suffix(name, "_jspb_proto", "_lib_proto"), +- srcs = srcs, +- deps = lib_proto_deps, +- visibility = visibility, +- ) +- js_proto_library( +- name = name, +- protos = [replace_suffix(name, "_jspb_proto", "_lib_proto")], +- output_mode = "NO_PREFIX_FLAT", +- # Need to specify this to work around bug in js_proto_library() +- # https://github.com/bazelbuild/rules_nodejs/issues/3503 +- legacy_path = "unused", +- deps = js_deps, +- visibility = visibility, +- ) ++ """OVMS build stub - JS proto generation is unused; keep signature only.""" ++ _ignore = [name, srcs, deps, lib_proto_deps, visibility, testonly, compatible_with] + + def mediapipe_js_proto_library(**kwargs): + mediapipe_js_proto_library_oss(**kwargs) +--- a/mediapipe/framework/port/build_config.bzl ++++ b/mediapipe/framework/port/build_config.bzl +@@ -3,7 +3,6 @@ + + """.bzl file for mediapipe open source build configs.""" + +-load("@aspect_rules_ts//ts:defs.bzl", "ts_project") + load( + "//mediapipe/framework/tool:mediapipe_proto.bzl", + _mediapipe_cc_proto_library = "mediapipe_cc_proto_library", +@@ -28,48 +27,8 @@ + deps = [], + testonly = 0, + allow_unoptimized_namespaces = False): +- """Generate ts_project for MediaPipe open source version. +- +- Args: +- name: The name of the target. +- srcs: The list of source files. +- visibility: The visibility of the target. +- deps: The list of dependencies. +- testonly: Whether the target is testonly. +- allow_unoptimized_namespaces: Whether to allow unoptimized namespaces. +- """ +- _ignore = [allow_unoptimized_namespaces] # buildifier: disable=unused-variable +- +- # aspect_rules_ts automatically generates a `{name}_types` +- # target when `declaration = True`. By suffixing the underlying ts_project +- # target name and exposing an alias, we dodge collisions with existing +- # explicit `_types` targets (like ts_declaration) defined in MediaPipe +- # BUILDs. +- internal_name = name + "_internal" +- +- ts_project(**provided_args( +- name = internal_name, +- srcs = srcs, +- visibility = visibility, +- deps = deps + [ +- "//:node_modules/@types/jasmine", +- "//:node_modules/@types/node", +- "//:node_modules/@types/offscreencanvas", +- "//:node_modules/@types/google-protobuf", +- "//:node_modules/@webgpu/types", +- ], +- testonly = testonly, +- declaration = True, +- transpiler = "tsc", +- tsconfig = "//:tsconfig", +- )) +- +- # Proxy the internal target back to the requested name. +- native.alias( +- name = name, +- actual = internal_name, +- visibility = visibility, +- ) ++ """OVMS build stub - TypeScript targets are unused.""" ++ _ignore = [name, srcs, visibility, deps, testonly, allow_unoptimized_namespaces] + + def ts_declaration( + name, +--- a/mediapipe/framework/tool/mediapipe_graph.bzl ++++ b/mediapipe/framework/tool/mediapipe_graph.bzl +@@ -25,7 +25,19 @@ + # buildifier: disable=out-of-order-load + # buildifier: disable=same-origin-load + load("//mediapipe/framework/deps:descriptor_set.bzl", "direct_descriptor_set", "transitive_descriptor_set") +-load("@litert//tflite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite") ++# OVMS stub - no tflite integration is used; delegate to plain native.cc_library. ++def cc_library_with_tflite(name, srcs = [], hdrs = [], tflite_deps = [], deps = [], alwayslink = None, visibility = None, testonly = None, **kwargs): ++ _ignore = [tflite_deps] ++ native.cc_library( ++ name = name, ++ srcs = srcs, ++ hdrs = hdrs, ++ deps = deps, ++ alwayslink = alwayslink, ++ visibility = visibility, ++ testonly = testonly, ++ **kwargs ++ ) + + def mediapipe_binary_graph(name, graph = None, output_name = None, deps = [], testonly = None, compatible_with = None, **kwargs): + """Converts a graph from text format to binary format. +--- a/mediapipe/gpu/BUILD ++++ b/mediapipe/gpu/BUILD +@@ -16,8 +16,7 @@ + load("@rules_cc//cc:objc_library.bzl", "objc_library") + load("@rules_cc//cc:cc_library.bzl", "cc_library") + load("@bazel_skylib//lib:selects.bzl", "selects") # buildifier: disable=out-of-order-load +-load("@build_bazel_rules_apple//apple:resources.bzl", metal_library = "apple_metal_library") +-load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") ++load("@mediapipe//mediapipe/gpu:ovms_apple_stub.bzl", metal_library = "apple_metal_library", ios_unit_test = "ios_unit_test") + load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") # buildifier: disable=out-of-order-load + load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test") # buildifier: disable=out-of-order-load + load("//mediapipe/framework:more_selects.bzl", "more_selects") +diff --git a/mediapipe/gpu/ovms_apple_stub.bzl b/mediapipe/gpu/ovms_apple_stub.bzl +new file mode 100644 +index 0000000..1111111 +--- /dev/null ++++ b/mediapipe/gpu/ovms_apple_stub.bzl +@@ -0,0 +1,14 @@ ++"""OVMS stubs for iOS/Apple-only rules used inside mediapipe/gpu/BUILD. ++ ++OVMS does not target Apple platforms, so these rules are unused at build ++time. Providing no-op definitions keeps mediapipe/gpu/BUILD parseable ++without pulling in @build_bazel_rules_apple. ++""" ++ ++def apple_metal_library(name, **kwargs): ++ _ignore = kwargs ++ native.filegroup(name = name, srcs = []) ++ ++def ios_unit_test(name, **kwargs): ++ _ignore = kwargs ++ native.filegroup(name = name, srcs = []) +--- a/mediapipe/framework/deps/status_macros.h ++++ b/mediapipe/framework/deps/status_macros.h +@@ -85,6 +85,7 @@ + // ABSL_RETURN_IF_ERROR(foo.Method(args...)); + // return absl::OkStatus(); + // } ++#undef ABSL_RETURN_IF_ERROR + #define ABSL_RETURN_IF_ERROR(expr) \ + MP_STATUS_MACROS_IMPL_ELSE_BLOCKER_ \ + if (mediapipe::status_macro_internal::StatusAdaptorForMacros \ +@@ -138,6 +139,7 @@ + // Example: Logging the error on failure. + // ABSL_ASSIGN_OR_RETURN(ValueType value, MaybeGetValue(query), _.LogError()); + // ++#undef ABSL_ASSIGN_OR_RETURN + #define ABSL_ASSIGN_OR_RETURN(...) \ + MP_STATUS_MACROS_IMPL_GET_VARIADIC_( \ + (__VA_ARGS__, MP_STATUS_MACROS_IMPL_ASSIGN_OR_RETURN_3_, \ +--- a/mediapipe/calculators/tensor/BUILD ++++ b/mediapipe/calculators/tensor/BUILD +@@ -14,7 +14,7 @@ + # + + load("@bazel_skylib//lib:selects.bzl", "selects") +-load("@litert//tflite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite") ++load("@mediapipe//mediapipe/gpu:ovms_apple_stub.bzl", cc_library_with_tflite = "apple_metal_library") + load("@rules_cc//cc:cc_library.bzl", "cc_library") + load("@rules_cc//cc:cc_test.bzl", "cc_test") + load("@rules_cc//cc:objc_library.bzl", "objc_library") +--- a/mediapipe/framework/formats/motion/optical_flow_field.h ++++ b/mediapipe/framework/formats/motion/optical_flow_field.h +@@ -24,7 +24,6 @@ + #include "mediapipe/framework/formats/motion/optical_flow_field_data.pb.h" + #include "mediapipe/framework/port/opencv_imgproc_inc.h" + #include "mediapipe/framework/port/status.h" +-#include "tensorflow/core/framework/tensor.h" + + namespace mediapipe { + +@@ -67,9 +66,6 @@ + const cv::Mat& flow_data() const { return flow_data_; } + cv::Mat& mutable_flow_data() { return flow_data_; } + +- // Converts from a tensorflow H x W x 2 float Tensor. The internal storage +- // for the optical flow field is reallocated. +- void CopyFromTensor(const tensorflow::Tensor& tensor); + + // Converts to/from associated proto. + void SetFromProto(const OpticalFlowFieldData& proto); +--- a/mediapipe/framework/formats/motion/optical_flow_field.cc ++++ b/mediapipe/framework/formats/motion/optical_flow_field.cc +@@ -147,22 +147,6 @@ + } + } + +-void OpticalFlowField::CopyFromTensor(const tensorflow::Tensor& tensor) { +- ABSL_CHECK_EQ(tensorflow::DT_FLOAT, tensor.dtype()); +- ABSL_CHECK_EQ(3, tensor.dims()) << "Tensor must be height x width x 2."; +- ABSL_CHECK_EQ(2, tensor.dim_size(2)) << "Tensor must be height x width x 2."; +- const int height = tensor.dim_size(0); +- const int width = tensor.dim_size(1); +- Allocate(width, height); +- typename tensorflow::TTypes::ConstTensor input_flow = +- tensor.shaped({height, width, 2}); +- for (int r = 0; r < height; ++r) { +- for (int c = 0; c < width; ++c) { +- flow_data_(r, c) = cv::Point2f(input_flow(r, c, 0), input_flow(r, c, 1)); +- } +- } +-} +- + void OpticalFlowField::SetFromProto(const OpticalFlowFieldData& proto) { + ABSL_CHECK_EQ(proto.width() * proto.height(), proto.dx_size()); + ABSL_CHECK_EQ(proto.width() * proto.height(), proto.dy_size()); +--- a/mediapipe/framework/formats/motion/BUILD ++++ b/mediapipe/framework/formats/motion/BUILD +@@ -48,7 +48,6 @@ + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings", +- "@org_tensorflow//tensorflow/core:framework", + ], + alwayslink = 1, + ) diff --git a/third_party/mediapipe_calculators/BUILD b/third_party/mediapipe_calculators/BUILD index 9a48658efc..4aeaf05d7b 100644 --- a/third_party/mediapipe_calculators/BUILD +++ b/third_party/mediapipe_calculators/BUILD @@ -29,15 +29,16 @@ cc_library( #ENABLE_WTIH_OV_CALCULATORS"@mediapipe//mediapipe/calculators/openvino:openvino_inference_calculator", #ENABLE_WTIH_OV_CALCULATORS"@mediapipe//mediapipe/calculators/openvino:openvino_tensors_to_classification_calculator", #ENABLE_WTIH_OV_CALCULATORS"@mediapipe//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator", - "@mediapipe//mediapipe/calculators/core:concatenate_vector_calculator", + # Upstream mediapipe pulls @litert/TFLite through these calculators; OVMS never runs TFLite. + #TFLITE"@mediapipe//mediapipe/calculators/core:concatenate_vector_calculator", "@mediapipe//mediapipe/calculators/core:flow_limiter_calculator", "@mediapipe//mediapipe/calculators/core:previous_loopback_calculator", - "@mediapipe//mediapipe/calculators/core:split_vector_calculator", + #TFLITE"@mediapipe//mediapipe/calculators/core:split_vector_calculator", "@mediapipe//mediapipe/calculators/core:add_header_calculator", "@mediapipe//mediapipe/calculators/core:begin_loop_calculator", - "@mediapipe//mediapipe/calculators/core:end_loop_calculator", - "@mediapipe//mediapipe/calculators/core:concatenate_vector_calculator_hdr", - "@mediapipe//mediapipe/calculators/core:concatenate_detection_vector_calculator", + #TFLITE"@mediapipe//mediapipe/calculators/core:end_loop_calculator", + #TFLITE"@mediapipe//mediapipe/calculators/core:concatenate_vector_calculator_hdr", + #TFLITE"@mediapipe//mediapipe/calculators/core:concatenate_detection_vector_calculator", "@mediapipe//mediapipe/calculators/core:concatenate_proto_list_calculator", "@mediapipe//mediapipe/calculators/core:clip_vector_size_calculator", #SEGFAULT_ON_GLOG_INIT"@mediapipe//mediapipe/calculators/core:clip_detection_vector_size_calculator", @@ -90,26 +91,27 @@ cc_library( "@mediapipe//mediapipe/calculators/image:image_clone_calculator", "@mediapipe//mediapipe/calculators/image:image_properties_calculator", #GPU"@mediapipe//mediapipe/calculators/image:mask_overlay_calculator", - "@mediapipe//mediapipe/calculators/image:feature_detector_calculator", + #TFLITE"@mediapipe//mediapipe/calculators/image:feature_detector_calculator", #REQUIRED_easyexif_DEPENDENCY"@mediapipe//mediapipe/calculators/image:image_file_properties_calculator", "@mediapipe//mediapipe/calculators/image:segmentation_smoothing_calculator", "@mediapipe//mediapipe/calculators/image:warp_affine_calculator", #REQUIRED_libyuv_DEPENDENCY"@mediapipe//mediapipe/calculators/image:yuv_to_image_calculator", #VISIBILITY_PRIVATE_THUS_SINK_BELOW"@mediapipe//mediapipe/calculators/internal:callback_packet_calculator", "@mediapipe//mediapipe/framework/tool:sink", - "@mediapipe//mediapipe/calculators/video:flow_to_image_calculator", - "@mediapipe//mediapipe/calculators/video:motion_analysis_calculator", - "@mediapipe//mediapipe/calculators/video:flow_packager_calculator", - "@mediapipe//mediapipe/calculators/video:box_tracker_calculator", - "@mediapipe//mediapipe/calculators/video:box_detector_calculator", - "@mediapipe//mediapipe/calculators/video:tracked_detection_manager_calculator", - "@mediapipe//mediapipe/calculators/video:video_pre_stream_calculator", + # Upstream mediapipe's video calculators use glog-only QCHECK/QCHECK_GT which are absent in absl-only ports. + #VIDEO"@mediapipe//mediapipe/calculators/video:flow_to_image_calculator", + #VIDEO"@mediapipe//mediapipe/calculators/video:motion_analysis_calculator", + #VIDEO"@mediapipe//mediapipe/calculators/video:flow_packager_calculator", + #VIDEO"@mediapipe//mediapipe/calculators/video:box_tracker_calculator", + #VIDEO"@mediapipe//mediapipe/calculators/video:box_detector_calculator", + #VIDEO"@mediapipe//mediapipe/calculators/video:tracked_detection_manager_calculator", + #VIDEO"@mediapipe//mediapipe/calculators/video:video_pre_stream_calculator", ] + select({ "//conditions:default": [ "@mediapipe//mediapipe/calculators/core:packet_cloner_calculator", # TODO windows: stdc++20 required - "@mediapipe//mediapipe/calculators/video:tvl1_optical_flow_calculator", # TODO windows: 'opencv2/optflow.hpp': No such file - will be available with opencv cmake on windows - "@mediapipe//mediapipe/calculators/video:opencv_video_decoder_calculator", - "@mediapipe//mediapipe/calculators/video:opencv_video_encoder_calculator", + #VIDEO"@mediapipe//mediapipe/calculators/video:tvl1_optical_flow_calculator", # TODO windows: 'opencv2/optflow.hpp': No such file - will be available with opencv cmake on windows + #VIDEO"@mediapipe//mediapipe/calculators/video:opencv_video_decoder_calculator", + #VIDEO"@mediapipe//mediapipe/calculators/video:opencv_video_encoder_calculator", ], "@ovms//src:windows" : [], }), diff --git a/third_party/model_api/BUILD b/third_party/model_api/BUILD new file mode 100644 index 0000000000..68e3f96fac --- /dev/null +++ b/third_party/model_api/BUILD @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023-2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +exports_files(["model_api_json_archive.patch"]) diff --git a/third_party/model_api/model_api.bzl b/third_party/model_api/model_api.bzl new file mode 100644 index 0000000000..8cdced29d6 --- /dev/null +++ b/third_party/model_api/model_api.bzl @@ -0,0 +1,158 @@ +# +# Copyright (c) 2023-2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") + +def _is_windows(ctx): + return ctx.os.name.lower().find("windows") != -1 + +def _get_windows_build_file(): + build_file_content = """ +load("@rules_foreign_cc//foreign_cc:cmake.bzl", "cmake") + +visibility = ["//visibility:public"] + +filegroup( + name = "all_srcs", + srcs = glob(["src/cpp/**"]), + visibility = ["//visibility:public"], +) + +cmake( + name = "model_api_cmake", + build_args = [ + "--verbose", + "-j 24", + ], + cache_entries = {{ + "CMAKE_POSITION_INDEPENDENT_CODE": "ON", + "ENABLE_PY_BINDINGS": "OFF", + "OpenVINO_DIR": "{openvino_dir}", + "OpenCV_DIR": "{opencv_dir}", + }}, + env = {{ + "HTTP_PROXY": "{http_proxy}", + "HTTPS_PROXY": "{https_proxy}", + }}, + lib_source = ":all_srcs", + out_static_libs = ["model_api.lib"], + tags = ["requires-network"], + deps = [ + "@mediapipe//mediapipe/framework/port:opencv_core", + "@windows_openvino//:openvino_new_headers", + ], +) + +cc_library( + name = "model_api", + deps = [ + "@mediapipe//mediapipe/framework/port:opencv_core", + "@windows_openvino//:openvino_new_headers", + ":model_api_cmake", + ], + visibility = ["//visibility:public"], +) +""" + + return build_file_content + +def _get_linux_build_file(): + build_file_content = """ +load("@rules_foreign_cc//foreign_cc:cmake.bzl", "cmake") + +visibility = ["//visibility:public"] + +filegroup( + name = "all_srcs", + srcs = glob(["src/cpp/**"]), + visibility = ["//visibility:public"], +) + +cmake( + name = "model_api_cmake", + build_args = [ + "--verbose", + "--", # <- Pass remaining options to the native tool. + # https://github.com/bazelbuild/rules_foreign_cc/issues/329 + # there is no elegant paralell compilation support + "VERBOSE=1", + "-j 24", + ], + cache_entries = {{ + "CMAKE_POSITION_INDEPENDENT_CODE": "ON", + "ENABLE_PY_BINDINGS": "OFF", + "OpenVINO_DIR": "/opt/intel/openvino/runtime/cmake", + }}, + env = {{ + "HTTP_PROXY": "{http_proxy}", + "HTTPS_PROXY": "{https_proxy}", + "http_proxy": "{http_proxy}", + "https_proxy": "{https_proxy}", + }}, + lib_source = ":all_srcs", + out_static_libs = ["libmodel_api.a"], + tags = ["requires-network"], + deps = [ + "@mediapipe//mediapipe/framework/port:opencv_core", + "@ovms//third_party:openvino", + ], +) + +cc_library( + name = "model_api", + deps = [ + "@mediapipe//mediapipe/framework/port:opencv_core", + "@ovms//third_party:openvino", + ":model_api_cmake", + ], + visibility = ["//visibility:public"], +)""" + return build_file_content + +def _impl(repository_ctx): + http_proxy = repository_ctx.os.environ.get("HTTP_PROXY", "") + https_proxy = repository_ctx.os.environ.get("HTTPS_PROXY", "") + openvino_dir = repository_ctx.os.environ.get("OpenVINO_DIR", "") + opencv_dir = repository_ctx.os.environ.get("OpenCV_DIR", "") + if not http_proxy: + http_proxy = repository_ctx.os.environ.get("http_proxy", "") + if not https_proxy: + https_proxy = repository_ctx.os.environ.get("https_proxy", "") + + # Note we need to escape '{/}' by doubling them due to call to format + if _is_windows(repository_ctx): + openvino_dir = openvino_dir.replace("\\", "\\\\").replace("/", "\\\\") + build_file_content = _get_windows_build_file() + else: + build_file_content = _get_linux_build_file() + + repository_ctx.file("BUILD", build_file_content.format(http_proxy = http_proxy, https_proxy = https_proxy, openvino_dir = openvino_dir, opencv_dir = opencv_dir)) + +model_api_repository = repository_rule( + implementation = _impl, + local = False, +) + +def workspace_model_api(): + model_api_repository(name = "_model-api") + new_git_repository( + name = "model_api", + remote = "https://github.com/openvinotoolkit/model_api/", + build_file = "@_model-api//:BUILD", + commit = "59685a8839176109ad677320228e2ee3ff94c788", # 26.11.2025 top of 'classic_cpp_support' branch + patch_args = ["-p1"], + patches = [Label("@ovms//third_party/model_api:model_api_json_archive.patch")], + ) diff --git a/third_party/model_api/model_api_json_archive.patch b/third_party/model_api/model_api_json_archive.patch new file mode 100644 index 0000000000..67b587cf44 --- /dev/null +++ b/third_party/model_api/model_api_json_archive.patch @@ -0,0 +1,19 @@ +diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt +index 0f822bc..43d1cc0 100644 +--- a/src/cpp/CMakeLists.txt ++++ b/src/cpp/CMakeLists.txt +@@ -35,9 +35,11 @@ find_package(OpenVINO REQUIRED + COMPONENTS Runtime Threading) + + include(FetchContent) +-FetchContent_Declare(json GIT_REPOSITORY https://github.com/nlohmann/json.git +- GIT_TAG d41ca94fa85d5119852e2f7a3f94335cc7cb0486 # PR #4709, fixes cmake deprecation warnings +- ) ++FetchContent_Declare(json ++ URL https://codeload.github.com/nlohmann/json/tar.gz/d41ca94fa85d5119852e2f7a3f94335cc7cb0486 ++ URL_HASH SHA256=d107f80866a9940b1c1cc55e88629a8cd7e86d8b2991b38d9ff6404e49b7938e ++ DOWNLOAD_EXTRACT_TIMESTAMP TRUE ++) + FetchContent_MakeAvailable(json) + + file(GLOB MODELS_SOURCES ./models/src/*.cpp) diff --git a/third_party/protobuf/BUILD b/third_party/protobuf/BUILD new file mode 100644 index 0000000000..0199c874ea --- /dev/null +++ b/third_party/protobuf/BUILD @@ -0,0 +1,17 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +exports_files(glob(["*.diff", "*.patch"])) diff --git a/third_party/protobuf/com_google_protobuf_fixes.diff b/third_party/protobuf/com_google_protobuf_fixes.diff new file mode 100644 index 0000000000..c8d4f3259f --- /dev/null +++ b/third_party/protobuf/com_google_protobuf_fixes.diff @@ -0,0 +1,92 @@ +diff --git a/BUILD b/BUILD +index 1690d4219..e13ca8338 100644 +--- a/BUILD ++++ b/BUILD +@@ -19,7 +19,7 @@ exports_files(["LICENSE"]) + # ZLIB configuration + ################################################################################ + +-ZLIB_DEPS = ["@zlib//:zlib"] ++ZLIB_DEPS = ["@zlib"] + + ################################################################################ + # Protobuf Runtime Library +@@ -197,6 +197,7 @@ cc_library( + includes = ["src/"], + linkopts = LINK_OPTS, + visibility = ["//visibility:public"], ++ alwayslink = 1, + ) + + PROTOBUF_DEPS = select({ +@@ -271,6 +272,7 @@ cc_library( + linkopts = LINK_OPTS, + visibility = ["//visibility:public"], + deps = [":protobuf_lite"] + PROTOBUF_DEPS, ++ alwayslink = 1, + ) + + # This provides just the header files for use in projects that need to build +diff --git a/src/google/protobuf/io/gzip_stream.h b/src/google/protobuf/io/gzip_stream.h +index f0283e86f..436c6ce4b 100644 +--- a/src/google/protobuf/io/gzip_stream.h ++++ b/src/google/protobuf/io/gzip_stream.h +@@ -47,10 +47,13 @@ + #include + #include + #include +-#include + + #include + ++#if HAVE_ZLIB ++#include ++#endif // HAVE_ZLIB ++ + namespace google { + namespace protobuf { + namespace io { +@@ -76,8 +79,10 @@ class PROTOBUF_EXPORT GzipInputStream : public ZeroCopyInputStream { + virtual ~GzipInputStream(); + + // Return last error message or NULL if no error. ++ #if HAVE_ZLIB + inline const char* ZlibErrorMessage() const { return zcontext_.msg; } + inline int ZlibErrorCode() const { return zerror_; } ++ #endif // HAVE_ZLIB + + // implements ZeroCopyInputStream ---------------------------------- + bool Next(const void** data, int* size) override; +@@ -90,8 +95,10 @@ class PROTOBUF_EXPORT GzipInputStream : public ZeroCopyInputStream { + + ZeroCopyInputStream* sub_stream_; + ++ #if HAVE_ZLIB + z_stream zcontext_; + int zerror_; ++ #endif // HAVE_ZLIB + + void* output_buffer_; + void* output_position_; +@@ -143,8 +150,10 @@ class PROTOBUF_EXPORT GzipOutputStream : public ZeroCopyOutputStream { + virtual ~GzipOutputStream(); + + // Return last error message or NULL if no error. ++ #if HAVE_ZLIB + inline const char* ZlibErrorMessage() const { return zcontext_.msg; } + inline int ZlibErrorCode() const { return zerror_; } ++ #endif // HAVE_ZLIB + + // Flushes data written so far to zipped data in the underlying stream. + // It is the caller's responsibility to flush the underlying stream if +@@ -177,8 +186,10 @@ class PROTOBUF_EXPORT GzipOutputStream : public ZeroCopyOutputStream { + void* sub_data_; + int sub_data_size_; + ++ #if HAVE_ZLIB + z_stream zcontext_; + int zerror_; ++ #endif // HAVE_ZLIB + void* input_buffer_; + size_t input_buffer_length_; + From 301e8c7bd35914f5434540977a9ef7fc5bff46c7 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Fri, 18 Sep 2026 13:30:50 +0200 Subject: [PATCH 09/10] merge main --- WORKSPACE | 2 +- third_party/model_api/model_api.bzl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 1833614917..13de794c98 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -125,7 +125,7 @@ http_archive( # RapidJSON # Must be defined earlier than tensorflow_serving because TFS is using older rapidjson # Version must match openvino.genai -> jinja2cpp -> rapidjson -# git/Jinja2Cpp/thirdparty/internal_deps.cmake +# git/Jinja2Cpp/third_party/internal_deps.cmake # Date: Tue May 9 21:31:22 2023 +0000 Avoid ptrdiff between pointers to different allocations http_archive( name = "com_github_tencent_rapidjson", diff --git a/third_party/model_api/model_api.bzl b/third_party/model_api/model_api.bzl index 8cdced29d6..4dbad2f7c2 100644 --- a/third_party/model_api/model_api.bzl +++ b/third_party/model_api/model_api.bzl @@ -87,7 +87,7 @@ cmake( "--verbose", "--", # <- Pass remaining options to the native tool. # https://github.com/bazelbuild/rules_foreign_cc/issues/329 - # there is no elegant paralell compilation support + # there is no elegant parallel compilation support "VERBOSE=1", "-j 24", ], From 8d6348c4ecc4170d9c7d4b8119bf412d51992472 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Fri, 18 Sep 2026 14:59:51 +0200 Subject: [PATCH 10/10] skip patches tests --- ci/lib_search.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/lib_search.py b/ci/lib_search.py index cd583ea6c0..5f401e9416 100644 --- a/ci/lib_search.py +++ b/ci/lib_search.py @@ -83,6 +83,7 @@ def check_dir(start_dir): 'REST_age_gender.ipynb', '__pycache__', 'abseil_gcc_8.5_constant_expression.patch', + 'absl_constexpr_fix.patch', 'azure_sdk.patch', 'cb.patch', 'bazel-', @@ -109,6 +110,7 @@ def check_dir(start_dir): 'partial.patch', 'ovms_drogon_trantor.patch', 'gorilla.patch', + 'grpc_missing_algorithm_include.patch', 'opencv_cmake_flags.txt', 'ovms-c/dist', 'requirements.txt', @@ -155,6 +157,10 @@ def check_dir(start_dir): "ServingRuntime.yaml", "dummy_facebook_template.jinja", "mediapipe_model_api_openvino_windows.patch", + 'model_api_json_archive.patch', + 'com_google_protobuf_fixes.diff', + 'ovms_no_litert.diff', + 'ovms_strip.diff', ] exclude_directories = ['/dist/', 'release_files/thirdparty-licenses', 'extras/chat_template_examples', 'src/test/llm/chat_templates']