From 65b3db36462edfb91a2ea247df024976f024c6d1 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:03:13 +0300 Subject: [PATCH 01/52] Refactor main function and modularize codebase - Refactored the main function for improved readability and maintainability. - Extracted benchmarking, prediction, and utility functions into separate modules. - Created dedicated modules for ONNX, OpenVINO, and CUDA operations to enhance code organization. - Simplified CUDA configurations and added detailed comments for clarity. --- README.md | 10 +- benchmark/__init__.py | 0 benchmark/benchmark_models.py | 18 ++ benchmark/benchmark_utils.py | 121 +++++++++++ common/__init__.py | 0 common/utils.py | 54 +++++ prediction/__init__.py | 0 prediction/prediction_models.py | 19 ++ prediction/prediction_utils.py | 79 +++++++ src/benchmark.py | 3 +- src/main.py | 359 ++++---------------------------- 11 files changed, 342 insertions(+), 321 deletions(-) create mode 100644 benchmark/__init__.py create mode 100644 benchmark/benchmark_models.py create mode 100644 benchmark/benchmark_utils.py create mode 100644 common/__init__.py create mode 100644 common/utils.py create mode 100644 prediction/__init__.py create mode 100644 prediction/prediction_models.py create mode 100644 prediction/prediction_utils.py diff --git a/README.md b/README.md index bb10f69..2be52b3 100644 --- a/README.md +++ b/README.md @@ -6,20 +6,20 @@ 2. [Requirements](#requirements) - [Steps to Run](#steps-to-run) - [Example Command](#example-command) -5. [RESULTS](#results) ![Static Badge](https://img.shields.io/badge/update-orange) +3. [RESULTS](#results) ![Static Badge](https://img.shields.io/badge/update-orange) - [Results explanation](#results-explanation) - [Example Input](#example-input) -6. [Benchmark Implementation Details](#benchmark-implementation-details) ![New](https://img.shields.io/badge/-New-842E5B) +4. [Benchmark Implementation Details](#benchmark-implementation-details) ![New](https://img.shields.io/badge/-New-842E5B) - [PyTorch CPU & CUDA](#pytorch-cpu--cuda) - [TensorRT FP32 & FP16](#tensorrt-fp32--fp16) - [ONNX](#onnx) - [OpenVINO](#openvino) -7. [Used methodologies](#used-methodologies) ![New](https://img.shields.io/badge/-New-96E5FE) +5. [Used methodologies](#used-methodologies) ![New](https://img.shields.io/badge/-New-96E5FE) - [TensorRT Optimization](#tensorrt-optimization) - [ONNX Exporter](#onnx-exporter) - [OV Exporter](#ov-exporter) -10. [Author](#author) -11. [References](#references) +6. [Author](#author) +7. [References](#references) diff --git a/benchmark/__init__.py b/benchmark/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py new file mode 100644 index 0000000..71b791e --- /dev/null +++ b/benchmark/benchmark_models.py @@ -0,0 +1,18 @@ +from benchmark.benchmark_utils import run_benchmark +from src.benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark +import openvino as ov +import torch +import onnxruntime as ort + + +def benchmark_onnx_model(ort_session: ort.InferenceSession): + run_benchmark(None, None, None, ort_session, onnx=True) + + +def benchmark_ov_model(ov_model: ov.CompiledModel): + ov_benchmark = OVBenchmark(ov_model, input_shape=(1, 3, 224, 224)) + ov_benchmark.run() + + +def benchmark_cuda_model(cuda_model: torch.nn.Module, device: str, dtype: torch.dtype): + run_benchmark(cuda_model, device, dtype) diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py new file mode 100644 index 0000000..74ae67f --- /dev/null +++ b/benchmark/benchmark_utils.py @@ -0,0 +1,121 @@ +import logging + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns +from typing import Dict, Any +import torch +import onnxruntime as ort + +from src.benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark + + +def run_benchmark( + model: torch.nn.Module, + device: str, + dtype: torch.dtype, + ort_session: ort.InferenceSession = None, + onnx: bool = False, +) -> None: + """ + Run and log the benchmark for the given model, device, and dtype. + + :param onnx: + :param ort_session: + :param model: The model to be benchmarked. + :param device: The device to run the benchmark on ("cpu" or "cuda"). + :param dtype: The data type to be used in the benchmark (typically torch.float32 or torch.float16). + """ + if onnx: + logging.info(f"Running Benchmark for ONNX") + benchmark = ONNXBenchmark(ort_session, input_shape=(32, 3, 224, 224)) + else: + logging.info(f"Running Benchmark for {device.upper()}") + benchmark = PyTorchBenchmark(model, device=device, dtype=dtype) + benchmark.run() + + +def run_all_benchmarks( + models: Dict[str, Any], img_batch: np.ndarray +) -> Dict[str, float]: + """ + Run benchmarks for all models and return a dictionary of average inference times. + + :param models: Dictionary of models. Key is model type ("onnx", "ov", "pytorch", "trt_fp32", "trt_fp16"), value is the model. + :param img_batch: The batch of images to run the benchmark on. + :return: Dictionary of average inference times. Key is model type, value is average inference time. + """ + results = {} + + # ONNX benchmark + onnx_benchmark = ONNXBenchmark(models["onnx"], img_batch.shape) + avg_time_onnx = onnx_benchmark.run() + results["ONNX"] = avg_time_onnx + + # OpenVINO benchmark + ov_benchmark = OVBenchmark(models["ov"], img_batch.shape) + avg_time_ov = ov_benchmark.run() + results["OpenVINO"] = avg_time_ov + + # PyTorch + TRT benchmark + configs = [ + ("cpu", torch.float32, False), + ("cuda", torch.float32, False), + ("cuda", torch.float32, True), + ("cuda", torch.float16, True), + ] + for device, precision, is_trt in configs: + model_to_use = models["pytorch"].to(device) + + if not is_trt: + pytorch_benchmark = PyTorchBenchmark( + model_to_use, device=device, dtype=precision + ) + avg_time_pytorch = pytorch_benchmark.run() + results[f"PyTorch_{device}"] = avg_time_pytorch + + else: + # TensorRT benchmarks + if precision == torch.float32 or precision == torch.float16: + mode = "fp32" if precision == torch.float32 else "fp16" + trt_benchmark = PyTorchBenchmark( + models[f"trt_{mode}"], device=device, dtype=precision + ) + avg_time_trt = trt_benchmark.run() + results[f"TRT_{mode}"] = avg_time_trt + + return results + + +def plot_benchmark_results(results: Dict[str, float]): + """ + Plot the benchmark results using Seaborn. + + :param results: Dictionary of average inference times. Key is model type, value is average inference time. + """ + # Convert dictionary to two lists for plotting + models = list(results.keys()) + times = list(results.values()) + + # Create a DataFrame for plotting + data = pd.DataFrame({"Model": models, "Time": times}) + + # Sort the DataFrame by Time + data = data.sort_values("Time", ascending=True) + + # Plot + plt.figure(figsize=(10, 6)) + ax = sns.barplot(x=data["Time"], y=data["Model"], palette="rocket") + + # Adding the actual values on the bars + for index, value in enumerate(data["Time"]): + ax.text(value, index, f"{value:.2f} ms", color="black", ha="left", va="center") + + plt.xlabel("Average Inference Time (ms)") + plt.ylabel("Model Type") + plt.title("ResNet50 - Inference Benchmark Results") + + # Save the plot to a file + plt.savefig("./inference/plot.png", bbox_inches="tight") + plt.show() \ No newline at end of file diff --git a/common/__init__.py b/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/utils.py b/common/utils.py new file mode 100644 index 0000000..a190468 --- /dev/null +++ b/common/utils.py @@ -0,0 +1,54 @@ +import argparse +import openvino as ov +import torch +from model import ModelLoader +from onnx_exporter import ONNXExporter +from ov_exporter import OVExporter +import onnxruntime as ort + + +# Model Initialization Functions +def init_onnx_model(onnx_path: str, model_loader: ModelLoader, device: torch.device) -> ort.InferenceSession: + onnx_exporter = ONNXExporter(model_loader.model, device, onnx_path) + onnx_exporter.export_model() + return ort.InferenceSession(onnx_path, providers=["CPUExecutionProvider"]) + + +def init_ov_model(onnx_path: str) -> ov.CompiledModel: + ov_exporter = OVExporter(onnx_path) + return ov_exporter.export_model() + + +def init_cuda_model(model_loader: ModelLoader, device: torch.device, dtype: torch.dtype) -> torch.nn.Module: + cuda_model = model_loader.model.to(device) + if device == "cuda": + cuda_model = torch.jit.trace(cuda_model, [torch.randn((1, 3, 224, 224)).to(device)]) + return cuda_model + + +def parse_arguments(): + # Initialize ArgumentParser with description + parser = argparse.ArgumentParser(description="PyTorch Inference") + parser.add_argument( + "--image_path", + type=str, + default="./inference/cat3.jpg", + help="Path to the image to predict", + ) + parser.add_argument( + "--topk", type=int, default=5, help="Number of top predictions to show" + ) + parser.add_argument( + "--onnx_path", + type=str, + default="./inference/model.onnx", + help="Path where model in ONNX format will be exported", + ) + parser.add_argument( + "--mode", + choices=["onnx", "ov", "cuda", "all"], + required=True, + help="Mode for exporting and running the model. Choices are: onnx, ov, cuda or all.", + ) + + return parser.parse_args() diff --git a/prediction/__init__.py b/prediction/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prediction/prediction_models.py b/prediction/prediction_models.py new file mode 100644 index 0000000..b02b0be --- /dev/null +++ b/prediction/prediction_models.py @@ -0,0 +1,19 @@ +import onnxruntime as ort +import openvino as ov +import numpy as np +import torch +from typing import List +from prediction.prediction_utils import make_prediction + + +# Prediction Functions +def predict_onnx_model(ort_session: ort.InferenceSession, img_batch: np.ndarray, topk: int, categories: List[str]): + make_prediction(ort_session, img_batch.cpu().numpy(), topk, categories) + + +def predict_ov_model(ov_model: ov.CompiledModel, img_batch: np.ndarray, topk: int, categories: List[str]): + make_prediction(ov_model, img_batch.cpu().numpy(), topk, categories) + + +def predict_cuda_model(cuda_model: torch.nn.Module, img_batch: torch.Tensor, topk: int, categories: List[str], precision: torch.dtype): + make_prediction(cuda_model, img_batch, topk, categories, precision) \ No newline at end of file diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py new file mode 100644 index 0000000..44dddfd --- /dev/null +++ b/prediction/prediction_utils.py @@ -0,0 +1,79 @@ +import logging +from typing import List, Tuple, Union, Dict, Any +import openvino as ov +import torch +import onnxruntime as ort +import numpy as np + + +def make_prediction( + model: Union[torch.nn.Module, ort.InferenceSession, ov.CompiledModel], + img_batch: Union[torch.Tensor, np.ndarray], + topk: int, + categories: List[str], + precision: torch.dtype = None, +) -> None: + """ + Make and print predictions for the given model, img_batch, topk, and categories. + + :param model: The model (or ONNX Runtime InferenceSession) to make predictions with. + :param img_batch: The batch of images to make predictions on. + :param topk: The number of top predictions to show. + :param categories: The list of categories to label the predictions. + :param precision: The data type to be used for the predictions (typically torch.float32 or torch.float16) for PyTorch models. + """ + is_onnx_model = isinstance(model, ort.InferenceSession) + is_ov_model = isinstance(model, ov.CompiledModel) + + if is_onnx_model: + # Get the input name for the ONNX model. + input_name = model.get_inputs()[0].name + + # Run the model with the properly named input. + ort_inputs = {input_name: img_batch} + ort_outs = model.run(None, ort_inputs) + + # Assuming the model returns a list with one array of class probabilities. + if len(ort_outs) > 0: + prob = ort_outs[0] + + # Checking if prob has more than one dimension and selecting the right one. + if prob.ndim > 1: + prob = prob[0] + + # Apply Softmax to get probabilities + prob = np.exp(prob) / np.sum(np.exp(prob)) + elif is_ov_model: + # For OV, the input name is usually the first input + input_name = next(iter(model.inputs)) + outputs = model(inputs={input_name: img_batch}) + + # Assuming the model returns a dictionary with one key for class probabilities + prob_key = next(iter(outputs)) + prob = outputs[prob_key] + + # Apply Softmax to get probabilities + prob = np.exp(prob[0]) / np.sum(np.exp(prob[0])) + + else: # PyTorch Model + if isinstance(img_batch, np.ndarray): + img_batch = torch.tensor(img_batch) + else: + img_batch = img_batch.clone().to(precision) + model.eval() + with torch.no_grad(): + outputs = model(img_batch.to(precision)) + prob = torch.nn.functional.softmax(outputs[0], dim=0) + prob = prob.cpu().numpy() + + top_indices = prob.argsort()[-topk:][::-1] + top_probs = prob[top_indices] + + for i in range(topk): + probability = top_probs[i] + if is_onnx_model: + # Accessing the DataFrame by row number using .iloc[] + class_label = categories.iloc[top_indices[i]].item() + else: + class_label = categories[0][int(top_indices[i])] + logging.info(f"#{i + 1}: {int(probability * 100)}% {class_label}") \ No newline at end of file diff --git a/src/benchmark.py b/src/benchmark.py index cf24764..9e866ae 100644 --- a/src/benchmark.py +++ b/src/benchmark.py @@ -115,7 +115,6 @@ def __init__( self.nwarmup = nwarmup self.nruns = nruns - def run(self): print("Warming up ...") # Adjusting the batch size in the input shape to match the expected input size of the model. @@ -196,4 +195,4 @@ def run(self): avg_time = total_time / self.num_runs logging.info(f"Average inference time: {avg_time * 1000:.2f} ms") - return avg_time * 1000 \ No newline at end of file + return avg_time * 1000 diff --git a/src/main.py b/src/main.py index aed87bf..9ea0ac4 100644 --- a/src/main.py +++ b/src/main.py @@ -1,344 +1,75 @@ -import argparse -import os -import logging -import pandas as pd -import openvino as ov -import torch -import torch_tensorrt -from typing import List, Tuple, Union, Dict, Any -import onnxruntime as ort -import numpy as np -import seaborn as sns -import matplotlib.pyplot as plt - -from model import ModelLoader +from benchmark.benchmark_models import * +from benchmark.benchmark_utils import * +from common.utils import * from image_processor import ImageProcessor -from benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark -from onnx_exporter import ONNXExporter -from ov_exporter import OVExporter +from prediction.prediction_models import * +from src.model import ModelLoader # Configure logging logging.basicConfig(filename="model.log", level=logging.INFO) -def run_benchmark( - model: torch.nn.Module, - device: str, - dtype: torch.dtype, - ort_session: ort.InferenceSession = None, - onnx: bool = False, -) -> None: - """ - Run and log the benchmark for the given model, device, and dtype. - - :param onnx: - :param ort_session: - :param model: The model to be benchmarked. - :param device: The device to run the benchmark on ("cpu" or "cuda"). - :param dtype: The data type to be used in the benchmark (typically torch.float32 or torch.float16). - """ - if onnx: - logging.info(f"Running Benchmark for ONNX") - benchmark = ONNXBenchmark(ort_session, input_shape=(32, 3, 224, 224)) - else: - logging.info(f"Running Benchmark for {device.upper()}") - benchmark = PyTorchBenchmark(model, device=device, dtype=dtype) - benchmark.run() - - -def make_prediction( - model: Union[torch.nn.Module, ort.InferenceSession, ov.CompiledModel], - img_batch: Union[torch.Tensor, np.ndarray], - topk: int, - categories: List[str], - precision: torch.dtype = None, -) -> None: - """ - Make and print predictions for the given model, img_batch, topk, and categories. - - :param model: The model (or ONNX Runtime InferenceSession) to make predictions with. - :param img_batch: The batch of images to make predictions on. - :param topk: The number of top predictions to show. - :param categories: The list of categories to label the predictions. - :param precision: The data type to be used for the predictions (typically torch.float32 or torch.float16) for PyTorch models. - """ - is_onnx_model = isinstance(model, ort.InferenceSession) - is_ov_model = isinstance(model, ov.CompiledModel) - - if is_onnx_model: - # Get the input name for the ONNX model. - input_name = model.get_inputs()[0].name - - # Run the model with the properly named input. - ort_inputs = {input_name: img_batch} - ort_outs = model.run(None, ort_inputs) - - # Assuming the model returns a list with one array of class probabilities. - if len(ort_outs) > 0: - prob = ort_outs[0] - - # Checking if prob has more than one dimension and selecting the right one. - if prob.ndim > 1: - prob = prob[0] - - # Apply Softmax to get probabilities - prob = np.exp(prob) / np.sum(np.exp(prob)) - elif is_ov_model: - # For OV, the input name is usually the first input - input_name = next(iter(model.inputs)) - outputs = model(inputs={input_name: img_batch}) - - # Assuming the model returns a dictionary with one key for class probabilities - prob_key = next(iter(outputs)) - prob = outputs[prob_key] - - # Apply Softmax to get probabilities - prob = np.exp(prob[0]) / np.sum(np.exp(prob[0])) - - else: # PyTorch Model - if isinstance(img_batch, np.ndarray): - img_batch = torch.tensor(img_batch) - else: - img_batch = img_batch.clone().to(precision) - model.eval() - with torch.no_grad(): - outputs = model(img_batch.to(precision)) - prob = torch.nn.functional.softmax(outputs[0], dim=0) - prob = prob.cpu().numpy() - - top_indices = prob.argsort()[-topk:][::-1] - top_probs = prob[top_indices] - - for i in range(topk): - probability = top_probs[i] - if is_onnx_model: - # Accessing the DataFrame by row number using .iloc[] - class_label = categories.iloc[top_indices[i]].item() - else: - class_label = categories[0][int(top_indices[i])] - logging.info(f"#{i + 1}: {int(probability * 100)}% {class_label}") - - -def run_all_benchmarks( - models: Dict[str, Any], img_batch: np.ndarray -) -> Dict[str, float]: - """ - Run benchmarks for all models and return a dictionary of average inference times. - - :param models: Dictionary of models. Key is model type ("onnx", "ov", "pytorch", "trt_fp32", "trt_fp16"), value is the model. - :param img_batch: The batch of images to run the benchmark on. - :return: Dictionary of average inference times. Key is model type, value is average inference time. - """ - results = {} - - # ONNX benchmark - onnx_benchmark = ONNXBenchmark(models["onnx"], img_batch.shape) - avg_time_onnx = onnx_benchmark.run() - results["ONNX"] = avg_time_onnx - - # OpenVINO benchmark - ov_benchmark = OVBenchmark(models["ov"], img_batch.shape) - avg_time_ov = ov_benchmark.run() - results["OpenVINO"] = avg_time_ov - - # PyTorch + TRT benchmark - configs = [ - ("cpu", torch.float32, False), - ("cuda", torch.float32, False), - ("cuda", torch.float32, True), - ("cuda", torch.float16, True), - ] - for device, precision, is_trt in configs: - model_to_use = models["pytorch"].to(device) - - if not is_trt: - pytorch_benchmark = PyTorchBenchmark( - model_to_use, device=device, dtype=precision - ) - avg_time_pytorch = pytorch_benchmark.run() - results[f"PyTorch_{device}"] = avg_time_pytorch - - else: - # TensorRT benchmarks - if precision == torch.float32 or precision == torch.float16: - mode = "fp32" if precision == torch.float32 else "fp16" - trt_benchmark = PyTorchBenchmark( - models[f"trt_{mode}"], device=device, dtype=precision - ) - avg_time_trt = trt_benchmark.run() - results[f"TRT_{mode}"] = avg_time_trt - - return results - - -def plot_benchmark_results(results: Dict[str, float]): - """ - Plot the benchmark results using Seaborn. - - :param results: Dictionary of average inference times. Key is model type, value is average inference time. - """ - # Convert dictionary to two lists for plotting - models = list(results.keys()) - times = list(results.values()) - - # Create a DataFrame for plotting - data = pd.DataFrame({"Model": models, "Time": times}) - - # Sort the DataFrame by Time - data = data.sort_values("Time", ascending=True) - - # Plot - plt.figure(figsize=(10, 6)) - ax = sns.barplot(x=data["Time"], y=data["Model"], palette="rocket") - - # Adding the actual values on the bars - for index, value in enumerate(data["Time"]): - ax.text(value, index, f"{value:.2f} ms", color="black", ha="left", va="center") - - plt.xlabel("Average Inference Time (ms)") - plt.ylabel("Model Type") - plt.title("ResNet50 - Inference Benchmark Results") - - # Save the plot to a file - plt.savefig("./inference/plot.png", bbox_inches="tight") - plt.show() - - def main() -> None: """ Main function to run inference, benchmarks, and predictions on the model using provided image and optional parameters. """ - # Initialize ArgumentParser with description - parser = argparse.ArgumentParser(description="PyTorch Inference") - parser.add_argument( - "--image_path", - type=str, - default="./inference/cat3.jpg", - help="Path to the image to predict", - ) - parser.add_argument( - "--topk", type=int, default=5, help="Number of top predictions to show" - ) - parser.add_argument( - "--onnx_path", - type=str, - default="./inference/model.onnx", - help="Path where model in ONNX format will be exported", - ) - parser.add_argument( - "--mode", - choices=["onnx", "ov", "cuda", "all"], - required=True, - help="Mode for exporting and running the model. Choices are: onnx, ov, cuda or all.", - ) - - args = parser.parse_args() + args = parse_arguments() + # Model and Image Initialization models = {} - - # Setup device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - # Initialize model and image processor model_loader = ModelLoader(device=device) img_processor = ImageProcessor(img_path=args.image_path, device=device) img_batch = img_processor.process_image() - if args.mode == "onnx" or args.mode == "all": - onnx_path = args.onnx_path - - # Export the model to ONNX format using ONNXExporter - onnx_exporter = ONNXExporter(model_loader.model, device, onnx_path) - onnx_exporter.export_model() - - # Create ONNX Runtime session - ort_session = ort.InferenceSession( - onnx_path, providers=["CPUExecutionProvider"] - ) - - models["onnx"] = ort_session - - # Run benchmark - # run_benchmark(None, None, None, ort_session, onnx=True) - - # Make prediction - print(f"Making prediction with {ort.get_device()} for ONNX model") - make_prediction( - ort_session, - img_batch.cpu().numpy(), - topk=args.topk, - categories=model_loader.categories, - ) - if args.mode == "ov" or args.mode == "all": - # Export the ONNX model to OpenVINO - ov_exporter = OVExporter(args.onnx_path) - ov_model = ov_exporter.export_model() - - models["ov"] = ov_model - - # Benchmark the OpenVINO model - ov_benchmark = OVBenchmark(ov_model, input_shape=(1, 3, 224, 224)) - ov_benchmark.run() - - # Run inference using the OpenVINO model - img_batch_ov = ( - img_processor.process_image().cpu().numpy() - ) # Assuming batch size of 1 - print(f"Making prediction with OpenVINO model") - make_prediction( - ov_benchmark.compiled_model, - img_batch_ov, - topk=args.topk, - categories=model_loader.categories, - ) - if args.mode == "cuda" or args.mode == "all": - # Define configurations for which to run benchmarks and make predictions - configs = [ - ("cpu", torch.float32), - ("cuda", torch.float32), - ("cuda", torch.float16), + # ONNX + if args.mode in ["onnx", "all"]: + ort_session = init_onnx_model(args.onnx_path, model_loader, device) + benchmark_onnx_model(ort_session) + predict_onnx_model(ort_session, img_batch, args.topk, model_loader.categories) + + # OpenVINO + if args.mode in ["ov", "all"]: + ov_model = init_ov_model(args.onnx_path) + benchmark_ov_model(ov_model) + predict_ov_model(ov_model, img_batch, args.topk, model_loader.categories) + + # CUDA + if args.mode in ["cuda", "all"]: + # CUDA configurations + cuda_configs = [ + {"device": "cpu", "precision": torch.float32, "is_trt": False}, + {"device": "cuda", "precision": torch.float32, "is_trt": False}, + {"device": "cuda", "precision": torch.float32, "is_trt": True}, + {"device": "cuda", "precision": torch.float16, "is_trt": True}, ] - for device, precision in configs: - model_to_use = model_loader.model.to(device) - models["pytorch"] = model_loader.model + for config in cuda_configs: + device = config["device"] + precision = config["precision"] + is_trt = config["is_trt"] - if device == "cuda": - print(f"Tracing {device} model") - model_to_use = torch.jit.trace( - model_to_use, [torch.randn((1, 3, 224, 224)).to(device)] - ) + model = init_cuda_model(model_loader, device, precision) - if precision == torch.float32 or precision == torch.float16: - print("Compiling TensorRT model") - model_to_use = torch_tensorrt.compile( - model_to_use, - inputs=[torch_tensorrt.Input((32, 3, 224, 224), dtype=precision)], - enabled_precisions={precision}, - truncate_long_and_double=True, - ) - if precision == torch.float32: - models["trt_fp32"] = model_to_use - else: - models["trt_fp16"] = model_to_use + # If the configuration is not for TensorRT, store the model under a PyTorch key + if not is_trt: + models[f"PyTorch_{device}"] = model + else: + # If it is for TensorRT, determine the mode (FP32 or FP16) and store under a TensorRT key + mode = "fp32" if precision == torch.float32 else "fp16" + models[f"trt_{mode}"] = model - """print(f"Making prediction with {device} model in {precision} precision") - make_prediction( - model_to_use, - img_batch.to(device), - args.topk, - model_loader.categories, - precision, + predict_cuda_model( + model, img_batch, args.topk, model_loader.categories, precision ) - print(f"Running Benchmark for {device} model in {precision} precision") - run_benchmark(model_to_use, device, precision) """ + # Aggregate Benchmark (if mode is "all") if args.mode == "all": - # Run all benchmarks - results = run_all_benchmarks(models, img_batch) + models["onnx"] = ort_session + models["ov"] = ov_model - # Plot results + results = run_all_benchmarks(models, img_batch) plot_benchmark_results(results) From a14095c2a2c2067ee81d18b85f57a41b33c642e4 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:06:47 +0300 Subject: [PATCH 02/52] Fixed imports --- benchmark/benchmark_models.py | 4 ++-- prediction/prediction_models.py | 2 +- src/main.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index 71b791e..de4cb1d 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -1,5 +1,5 @@ -from benchmark.benchmark_utils import run_benchmark -from src.benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark +from benchmark_utils import run_benchmark +from benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark import openvino as ov import torch import onnxruntime as ort diff --git a/prediction/prediction_models.py b/prediction/prediction_models.py index b02b0be..75a73ec 100644 --- a/prediction/prediction_models.py +++ b/prediction/prediction_models.py @@ -3,7 +3,7 @@ import numpy as np import torch from typing import List -from prediction.prediction_utils import make_prediction +from prediction_utils import make_prediction # Prediction Functions diff --git a/src/main.py b/src/main.py index 9ea0ac4..f6f8123 100644 --- a/src/main.py +++ b/src/main.py @@ -1,9 +1,9 @@ -from benchmark.benchmark_models import * -from benchmark.benchmark_utils import * -from common.utils import * +from benchmark_models import * +from benchmark_utils import * +from utils import * from image_processor import ImageProcessor -from prediction.prediction_models import * -from src.model import ModelLoader +from prediction_models import * +from model import ModelLoader # Configure logging logging.basicConfig(filename="model.log", level=logging.INFO) From 86540f944e8a175e0e0d266e3cf74547c3bd007d Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:16:47 +0300 Subject: [PATCH 03/52] Fixed imports and refactored --- benchmark/benchmark_models.py | 2 +- benchmark/benchmark_utils.py | 2 +- common/utils.py | 12 +++++++++--- prediction/prediction_models.py | 21 +++++++++++++++++---- prediction/prediction_utils.py | 2 +- src/main.py | 8 ++++---- 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index de4cb1d..d35e0dc 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -1,4 +1,4 @@ -from benchmark_utils import run_benchmark +from benchmark.benchmark_utils import run_benchmark from benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark import openvino as ov import torch diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index 74ae67f..15afea3 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -118,4 +118,4 @@ def plot_benchmark_results(results: Dict[str, float]): # Save the plot to a file plt.savefig("./inference/plot.png", bbox_inches="tight") - plt.show() \ No newline at end of file + plt.show() diff --git a/common/utils.py b/common/utils.py index a190468..5eb26e6 100644 --- a/common/utils.py +++ b/common/utils.py @@ -8,7 +8,9 @@ # Model Initialization Functions -def init_onnx_model(onnx_path: str, model_loader: ModelLoader, device: torch.device) -> ort.InferenceSession: +def init_onnx_model( + onnx_path: str, model_loader: ModelLoader, device: torch.device +) -> ort.InferenceSession: onnx_exporter = ONNXExporter(model_loader.model, device, onnx_path) onnx_exporter.export_model() return ort.InferenceSession(onnx_path, providers=["CPUExecutionProvider"]) @@ -19,10 +21,14 @@ def init_ov_model(onnx_path: str) -> ov.CompiledModel: return ov_exporter.export_model() -def init_cuda_model(model_loader: ModelLoader, device: torch.device, dtype: torch.dtype) -> torch.nn.Module: +def init_cuda_model( + model_loader: ModelLoader, device: torch.device, dtype: torch.dtype +) -> torch.nn.Module: cuda_model = model_loader.model.to(device) if device == "cuda": - cuda_model = torch.jit.trace(cuda_model, [torch.randn((1, 3, 224, 224)).to(device)]) + cuda_model = torch.jit.trace( + cuda_model, [torch.randn((1, 3, 224, 224)).to(device)] + ) return cuda_model diff --git a/prediction/prediction_models.py b/prediction/prediction_models.py index 75a73ec..929ad18 100644 --- a/prediction/prediction_models.py +++ b/prediction/prediction_models.py @@ -7,13 +7,26 @@ # Prediction Functions -def predict_onnx_model(ort_session: ort.InferenceSession, img_batch: np.ndarray, topk: int, categories: List[str]): +def predict_onnx_model( + ort_session: ort.InferenceSession, + img_batch: np.ndarray, + topk: int, + categories: List[str], +): make_prediction(ort_session, img_batch.cpu().numpy(), topk, categories) -def predict_ov_model(ov_model: ov.CompiledModel, img_batch: np.ndarray, topk: int, categories: List[str]): +def predict_ov_model( + ov_model: ov.CompiledModel, img_batch: np.ndarray, topk: int, categories: List[str] +): make_prediction(ov_model, img_batch.cpu().numpy(), topk, categories) -def predict_cuda_model(cuda_model: torch.nn.Module, img_batch: torch.Tensor, topk: int, categories: List[str], precision: torch.dtype): - make_prediction(cuda_model, img_batch, topk, categories, precision) \ No newline at end of file +def predict_cuda_model( + cuda_model: torch.nn.Module, + img_batch: torch.Tensor, + topk: int, + categories: List[str], + precision: torch.dtype, +): + make_prediction(cuda_model, img_batch, topk, categories, precision) diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index 44dddfd..6abab00 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -76,4 +76,4 @@ def make_prediction( class_label = categories.iloc[top_indices[i]].item() else: class_label = categories[0][int(top_indices[i])] - logging.info(f"#{i + 1}: {int(probability * 100)}% {class_label}") \ No newline at end of file + logging.info(f"#{i + 1}: {int(probability * 100)}% {class_label}") diff --git a/src/main.py b/src/main.py index f6f8123..34a1a91 100644 --- a/src/main.py +++ b/src/main.py @@ -1,8 +1,8 @@ -from benchmark_models import * -from benchmark_utils import * -from utils import * +from benchmark.benchmark_models import * +from benchmark.benchmark_utils import * +from common.utils import * from image_processor import ImageProcessor -from prediction_models import * +from prediction.prediction_models import * from model import ModelLoader # Configure logging From 4f5b75d6d4218e07006c598602621f04b707747b Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:20:08 +0300 Subject: [PATCH 04/52] Fixed imports and refactored --- src/main.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main.py b/src/main.py index 34a1a91..53c6f66 100644 --- a/src/main.py +++ b/src/main.py @@ -1,6 +1,13 @@ -from benchmark.benchmark_models import * -from benchmark.benchmark_utils import * -from common.utils import * +import logging + +from benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model +from benchmark.benchmark_utils import run_all_benchmarks, plot_benchmark_results +from common.utils import ( + parse_arguments, + init_onnx_model, + init_ov_model, + init_cuda_model, +) from image_processor import ImageProcessor from prediction.prediction_models import * from model import ModelLoader From bc67224c90e7eb5bcb4e0c4f14253f43ea77d361 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:24:09 +0300 Subject: [PATCH 05/52] Fixed imports and refactored --- benchmark/benchmark_models.py | 2 +- benchmark/benchmark_utils.py | 2 +- src/main.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index d35e0dc..de4cb1d 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -1,4 +1,4 @@ -from benchmark.benchmark_utils import run_benchmark +from benchmark_utils import run_benchmark from benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark import openvino as ov import torch diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index 15afea3..f0c2ec5 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -8,7 +8,7 @@ import torch import onnxruntime as ort -from src.benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark +from benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark def run_benchmark( diff --git a/src/main.py b/src/main.py index 53c6f66..503d4cf 100644 --- a/src/main.py +++ b/src/main.py @@ -1,15 +1,15 @@ import logging -from benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model -from benchmark.benchmark_utils import run_all_benchmarks, plot_benchmark_results -from common.utils import ( +from benchmark_models import benchmark_onnx_model, benchmark_ov_model +from benchmark_utils import run_all_benchmarks, plot_benchmark_results +from utils import ( parse_arguments, init_onnx_model, init_ov_model, init_cuda_model, ) from image_processor import ImageProcessor -from prediction.prediction_models import * +from prediction_models import * from model import ModelLoader # Configure logging From 0f59440104c224f898bc4688038a688ad4bf8e30 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:37:35 +0300 Subject: [PATCH 06/52] Fixed imports and refactored --- benchmark/benchmark_models.py | 2 +- prediction/prediction_models.py | 2 +- src/main.py | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index de4cb1d..d35e0dc 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -1,4 +1,4 @@ -from benchmark_utils import run_benchmark +from benchmark.benchmark_utils import run_benchmark from benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark import openvino as ov import torch diff --git a/prediction/prediction_models.py b/prediction/prediction_models.py index 929ad18..aaaf230 100644 --- a/prediction/prediction_models.py +++ b/prediction/prediction_models.py @@ -3,7 +3,7 @@ import numpy as np import torch from typing import List -from prediction_utils import make_prediction +from prediction.prediction_utils import make_prediction # Prediction Functions diff --git a/src/main.py b/src/main.py index 503d4cf..7d4ffff 100644 --- a/src/main.py +++ b/src/main.py @@ -1,19 +1,21 @@ import logging +import sys -from benchmark_models import benchmark_onnx_model, benchmark_ov_model -from benchmark_utils import run_all_benchmarks, plot_benchmark_results -from utils import ( +from benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model +from benchmark.benchmark_utils import run_all_benchmarks, plot_benchmark_results +from common.utils import ( parse_arguments, init_onnx_model, init_ov_model, init_cuda_model, ) from image_processor import ImageProcessor -from prediction_models import * +from prediction.prediction_models import * from model import ModelLoader # Configure logging logging.basicConfig(filename="model.log", level=logging.INFO) +sys.path.append('/workspace') def main() -> None: From a70efcd472c8884c8203d95ceaae1cbd42440061 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:41:24 +0300 Subject: [PATCH 07/52] Fixed imports and refactored --- benchmark/benchmark_models.py | 2 +- benchmark/benchmark_utils.py | 2 +- src/{benchmark.py => benchmark_class.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/{benchmark.py => benchmark_class.py} (100%) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index d35e0dc..c917a85 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -1,5 +1,5 @@ from benchmark.benchmark_utils import run_benchmark -from benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark +from benchmark_class import PyTorchBenchmark, ONNXBenchmark, OVBenchmark import openvino as ov import torch import onnxruntime as ort diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index f0c2ec5..80c540b 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -8,7 +8,7 @@ import torch import onnxruntime as ort -from benchmark import PyTorchBenchmark, ONNXBenchmark, OVBenchmark +from benchmark_class import PyTorchBenchmark, ONNXBenchmark, OVBenchmark def run_benchmark( diff --git a/src/benchmark.py b/src/benchmark_class.py similarity index 100% rename from src/benchmark.py rename to src/benchmark_class.py From 79bb3670d9010c91a331d1c61ab1502285bfd317 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:43:17 +0300 Subject: [PATCH 08/52] Fixed imports and refactored --- src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index 7d4ffff..cc14d32 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,7 @@ import logging import sys -from benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model +from ..benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model from benchmark.benchmark_utils import run_all_benchmarks, plot_benchmark_results from common.utils import ( parse_arguments, From 687791d6ca336dfb4cf8e9c9092ee86c462bcff7 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:48:15 +0300 Subject: [PATCH 09/52] Fixed imports and refactored --- src/main.py => main.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) rename src/main.py => main.py (93%) diff --git a/src/main.py b/main.py similarity index 93% rename from src/main.py rename to main.py index cc14d32..1987da1 100644 --- a/src/main.py +++ b/main.py @@ -1,7 +1,6 @@ import logging -import sys -from ..benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model +from benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model from benchmark.benchmark_utils import run_all_benchmarks, plot_benchmark_results from common.utils import ( parse_arguments, @@ -9,13 +8,12 @@ init_ov_model, init_cuda_model, ) -from image_processor import ImageProcessor +from src.image_processor import ImageProcessor from prediction.prediction_models import * -from model import ModelLoader +from src.model import ModelLoader # Configure logging logging.basicConfig(filename="model.log", level=logging.INFO) -sys.path.append('/workspace') def main() -> None: From fb27d3adf3ab86e7d121e52675ae6ef320186bac Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:51:02 +0300 Subject: [PATCH 10/52] Fixed imports and refactored --- benchmark/benchmark_models.py | 2 +- benchmark/benchmark_utils.py | 2 +- common/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index c917a85..483143d 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -1,5 +1,5 @@ from benchmark.benchmark_utils import run_benchmark -from benchmark_class import PyTorchBenchmark, ONNXBenchmark, OVBenchmark +from src.benchmark_class import PyTorchBenchmark, ONNXBenchmark, OVBenchmark import openvino as ov import torch import onnxruntime as ort diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index 80c540b..b6ba8fa 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -8,7 +8,7 @@ import torch import onnxruntime as ort -from benchmark_class import PyTorchBenchmark, ONNXBenchmark, OVBenchmark +from src.benchmark_class import PyTorchBenchmark, ONNXBenchmark, OVBenchmark def run_benchmark( diff --git a/common/utils.py b/common/utils.py index 5eb26e6..1d0743b 100644 --- a/common/utils.py +++ b/common/utils.py @@ -1,7 +1,7 @@ import argparse import openvino as ov import torch -from model import ModelLoader +from src.model import ModelLoader from onnx_exporter import ONNXExporter from ov_exporter import OVExporter import onnxruntime as ort From 7dfa3da5b19e0ad08534015ca062d557e868588d Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:52:26 +0300 Subject: [PATCH 11/52] Fixed imports and refactored --- common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/utils.py b/common/utils.py index 1d0743b..a5b9b4a 100644 --- a/common/utils.py +++ b/common/utils.py @@ -2,8 +2,8 @@ import openvino as ov import torch from src.model import ModelLoader -from onnx_exporter import ONNXExporter -from ov_exporter import OVExporter +from src.onnx_exporter import ONNXExporter +from src.ov_exporter import OVExporter import onnxruntime as ort From f13f6728e2b5ef795fc4e074505e32c6759d96e9 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 21:54:44 +0300 Subject: [PATCH 12/52] Fixed imports and refactored --- prediction/prediction_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index 6abab00..91495c3 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -60,7 +60,8 @@ def make_prediction( img_batch = torch.tensor(img_batch) else: img_batch = img_batch.clone().to(precision) - model.eval() + if isinstance(model, torch.nn.Module): # Check if the model is a PyTorch model + model.eval() with torch.no_grad(): outputs = model(img_batch.to(precision)) prob = torch.nn.functional.softmax(outputs[0], dim=0) From 2d01ce54fedb087b5d4ec1c15f58b1f68afbbb5c Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:00:19 +0300 Subject: [PATCH 13/52] Fixed imports and refactored --- benchmark/benchmark_models.py | 4 +++- main.py | 4 ++-- prediction/prediction_utils.py | 3 +-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index 483143d..e772360 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -1,3 +1,4 @@ +import src.benchmark_class from benchmark.benchmark_utils import run_benchmark from src.benchmark_class import PyTorchBenchmark, ONNXBenchmark, OVBenchmark import openvino as ov @@ -9,9 +10,10 @@ def benchmark_onnx_model(ort_session: ort.InferenceSession): run_benchmark(None, None, None, ort_session, onnx=True) -def benchmark_ov_model(ov_model: ov.CompiledModel): +def benchmark_ov_model(ov_model: ov.CompiledModel) -> src.benchmark_class.OVBenchmark: ov_benchmark = OVBenchmark(ov_model, input_shape=(1, 3, 224, 224)) ov_benchmark.run() + return ov_benchmark def benchmark_cuda_model(cuda_model: torch.nn.Module, device: str, dtype: torch.dtype): diff --git a/main.py b/main.py index 1987da1..4ed20de 100644 --- a/main.py +++ b/main.py @@ -39,8 +39,8 @@ def main() -> None: # OpenVINO if args.mode in ["ov", "all"]: ov_model = init_ov_model(args.onnx_path) - benchmark_ov_model(ov_model) - predict_ov_model(ov_model, img_batch, args.topk, model_loader.categories) + ov_benchmark = benchmark_ov_model(ov_model) + predict_ov_model(ov_benchmark.compiled_model, img_batch, args.topk, model_loader.categories) # CUDA if args.mode in ["cuda", "all"]: diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index 91495c3..6abab00 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -60,8 +60,7 @@ def make_prediction( img_batch = torch.tensor(img_batch) else: img_batch = img_batch.clone().to(precision) - if isinstance(model, torch.nn.Module): # Check if the model is a PyTorch model - model.eval() + model.eval() with torch.no_grad(): outputs = model(img_batch.to(precision)) prob = torch.nn.functional.softmax(outputs[0], dim=0) From 555bf6df95f0bf0844a3c8fbcfce847bdbb63dc2 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:02:25 +0300 Subject: [PATCH 14/52] Fixed imports and refactored --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 4ed20de..7f4dc1d 100644 --- a/main.py +++ b/main.py @@ -68,7 +68,7 @@ def main() -> None: models[f"trt_{mode}"] = model predict_cuda_model( - model, img_batch, args.topk, model_loader.categories, precision + model.to(device), img_batch.to(device), args.topk, model_loader.categories, precision ) # Aggregate Benchmark (if mode is "all") From 6bf5cb202345f61087da88b0020693205a6d3645 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:12:16 +0300 Subject: [PATCH 15/52] Fixed imports and refactored --- main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 7f4dc1d..9f79ef9 100644 --- a/main.py +++ b/main.py @@ -62,13 +62,15 @@ def main() -> None: # If the configuration is not for TensorRT, store the model under a PyTorch key if not is_trt: models[f"PyTorch_{device}"] = model + model = model.to(device) + img_batch = img_batch.to(device) else: # If it is for TensorRT, determine the mode (FP32 or FP16) and store under a TensorRT key mode = "fp32" if precision == torch.float32 else "fp16" models[f"trt_{mode}"] = model predict_cuda_model( - model.to(device), img_batch.to(device), args.topk, model_loader.categories, precision + model, img_batch, args.topk, model_loader.categories, precision ) # Aggregate Benchmark (if mode is "all") From 525278aba6512fe4793966f3ab042dcef14a5b73 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:15:55 +0300 Subject: [PATCH 16/52] Fixed imports and refactored --- main.py | 8 ++++++++ prediction/prediction_utils.py | 1 + 2 files changed, 9 insertions(+) diff --git a/main.py b/main.py index 9f79ef9..70c6b95 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ import logging +import torch_tensorrt from benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model from benchmark.benchmark_utils import run_all_benchmarks, plot_benchmark_results @@ -65,6 +66,13 @@ def main() -> None: model = model.to(device) img_batch = img_batch.to(device) else: + print("Compiling TensorRT model") + model_to_use = torch_tensorrt.compile( + model_to_use, + inputs=[torch_tensorrt.Input((1, 3, 224, 224), dtype=precision)], + enabled_precisions={precision}, + truncate_long_and_double=True, + ) # If it is for TensorRT, determine the mode (FP32 or FP16) and store under a TensorRT key mode = "fp32" if precision == torch.float32 else "fp16" models[f"trt_{mode}"] = model diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index 6abab00..72b06dd 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -77,3 +77,4 @@ def make_prediction( else: class_label = categories[0][int(top_indices[i])] logging.info(f"#{i + 1}: {int(probability * 100)}% {class_label}") + print(f"#{i + 1}: {int(probability * 100)}% {class_label}") From b523c5380e6d19dfef5fdcf26e660818e5cf64bf Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:17:33 +0300 Subject: [PATCH 17/52] Fixed imports and refactored --- main.py | 4 ++-- prediction/prediction_utils.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 70c6b95..5fb5b4a 100644 --- a/main.py +++ b/main.py @@ -68,14 +68,14 @@ def main() -> None: else: print("Compiling TensorRT model") model_to_use = torch_tensorrt.compile( - model_to_use, + model, inputs=[torch_tensorrt.Input((1, 3, 224, 224), dtype=precision)], enabled_precisions={precision}, truncate_long_and_double=True, ) # If it is for TensorRT, determine the mode (FP32 or FP16) and store under a TensorRT key mode = "fp32" if precision == torch.float32 else "fp16" - models[f"trt_{mode}"] = model + models[f"trt_{mode}"] = model_to_use predict_cuda_model( model, img_batch, args.topk, model_loader.categories, precision diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index 72b06dd..6abab00 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -77,4 +77,3 @@ def make_prediction( else: class_label = categories[0][int(top_indices[i])] logging.info(f"#{i + 1}: {int(probability * 100)}% {class_label}") - print(f"#{i + 1}: {int(probability * 100)}% {class_label}") From 81fa6e496af2554de3364932d3bdf5d6ee4ee4e8 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:22:02 +0300 Subject: [PATCH 18/52] Fixed imports and refactored --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 5fb5b4a..bf1f471 100644 --- a/main.py +++ b/main.py @@ -67,7 +67,7 @@ def main() -> None: img_batch = img_batch.to(device) else: print("Compiling TensorRT model") - model_to_use = torch_tensorrt.compile( + model = torch_tensorrt.compile( model, inputs=[torch_tensorrt.Input((1, 3, 224, 224), dtype=precision)], enabled_precisions={precision}, @@ -75,7 +75,7 @@ def main() -> None: ) # If it is for TensorRT, determine the mode (FP32 or FP16) and store under a TensorRT key mode = "fp32" if precision == torch.float32 else "fp16" - models[f"trt_{mode}"] = model_to_use + models[f"trt_{mode}"] = model predict_cuda_model( model, img_batch, args.topk, model_loader.categories, precision From 85486a54b71bcb3e85de0b5b62e730f3163c9af3 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:24:56 +0300 Subject: [PATCH 19/52] Fixed imports and refactored --- common/utils.py | 2 +- prediction/prediction_utils.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/common/utils.py b/common/utils.py index a5b9b4a..cd3e120 100644 --- a/common/utils.py +++ b/common/utils.py @@ -53,7 +53,7 @@ def parse_arguments(): parser.add_argument( "--mode", choices=["onnx", "ov", "cuda", "all"], - required=True, + default="all", help="Mode for exporting and running the model. Choices are: onnx, ov, cuda or all.", ) diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index 6abab00..a9ea429 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -26,6 +26,7 @@ def make_prediction( is_ov_model = isinstance(model, ov.CompiledModel) if is_onnx_model: + logging.info(f"Running prediction for ONNX model") # Get the input name for the ONNX model. input_name = model.get_inputs()[0].name @@ -44,6 +45,7 @@ def make_prediction( # Apply Softmax to get probabilities prob = np.exp(prob) / np.sum(np.exp(prob)) elif is_ov_model: + logging.info(f"Running prediction for OV model") # For OV, the input name is usually the first input input_name = next(iter(model.inputs)) outputs = model(inputs={input_name: img_batch}) @@ -56,6 +58,7 @@ def make_prediction( prob = np.exp(prob[0]) / np.sum(np.exp(prob[0])) else: # PyTorch Model + logging.info(f"Running prediction for PyTorch model") if isinstance(img_batch, np.ndarray): img_batch = torch.tensor(img_batch) else: From b734a87ca7f243daa304ce92e69980263775cba7 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:28:13 +0300 Subject: [PATCH 20/52] Fixed imports and refactored --- benchmark/benchmark_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index b6ba8fa..3c7ec0e 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -31,7 +31,7 @@ def run_benchmark( logging.info(f"Running Benchmark for ONNX") benchmark = ONNXBenchmark(ort_session, input_shape=(32, 3, 224, 224)) else: - logging.info(f"Running Benchmark for {device.upper()}") + logging.info(f"Running Benchmark for {device.upper()} and precision {dtype}") benchmark = PyTorchBenchmark(model, device=device, dtype=dtype) benchmark.run() @@ -66,7 +66,7 @@ def run_all_benchmarks( ("cuda", torch.float16, True), ] for device, precision, is_trt in configs: - model_to_use = models["pytorch"].to(device) + model_to_use = models[f"PyTorch_{device}"].to(device) if not is_trt: pytorch_benchmark = PyTorchBenchmark( From 9d762545eb49a2cb1f14bcacf4d2452646cd5602 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:35:59 +0300 Subject: [PATCH 21/52] Fixed imports and refactored --- main.py | 19 +++++++++++-------- prediction/prediction_models.py | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index bf1f471..3ccb895 100644 --- a/main.py +++ b/main.py @@ -34,14 +34,16 @@ def main() -> None: # ONNX if args.mode in ["onnx", "all"]: ort_session = init_onnx_model(args.onnx_path, model_loader, device) - benchmark_onnx_model(ort_session) - predict_onnx_model(ort_session, img_batch, args.topk, model_loader.categories) + if args.mode != "all": + benchmark_onnx_model(ort_session) + predict_onnx_model(ort_session, img_batch, args.topk, model_loader.categories) # OpenVINO if args.mode in ["ov", "all"]: ov_model = init_ov_model(args.onnx_path) - ov_benchmark = benchmark_ov_model(ov_model) - predict_ov_model(ov_benchmark.compiled_model, img_batch, args.topk, model_loader.categories) + if args.mode != "all": + ov_benchmark = benchmark_ov_model(ov_model) + predict_ov_model(ov_benchmark.compiled_model, img_batch, args.topk, model_loader.categories) # CUDA if args.mode in ["cuda", "all"]: @@ -69,7 +71,7 @@ def main() -> None: print("Compiling TensorRT model") model = torch_tensorrt.compile( model, - inputs=[torch_tensorrt.Input((1, 3, 224, 224), dtype=precision)], + inputs=[torch_tensorrt.Input((32, 3, 224, 224), dtype=precision)], enabled_precisions={precision}, truncate_long_and_double=True, ) @@ -77,9 +79,10 @@ def main() -> None: mode = "fp32" if precision == torch.float32 else "fp16" models[f"trt_{mode}"] = model - predict_cuda_model( - model, img_batch, args.topk, model_loader.categories, precision - ) + if args.mode != "all": + predict_cuda_model( + model, img_batch, args.topk, model_loader.categories, precision + ) # Aggregate Benchmark (if mode is "all") if args.mode == "all": diff --git a/prediction/prediction_models.py b/prediction/prediction_models.py index aaaf230..ad1d4de 100644 --- a/prediction/prediction_models.py +++ b/prediction/prediction_models.py @@ -17,7 +17,7 @@ def predict_onnx_model( def predict_ov_model( - ov_model: ov.CompiledModel, img_batch: np.ndarray, topk: int, categories: List[str] + ov_model: ov.CompiledModel, img_batch: np.ndarray, topk: int, categories: List[str], to_skip: bool = False ): make_prediction(ov_model, img_batch.cpu().numpy(), topk, categories) From 071ed43bb456653c5bd828098de5328aa3740da8 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:36:14 +0300 Subject: [PATCH 22/52] Fixed imports and refactored --- prediction/prediction_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prediction/prediction_models.py b/prediction/prediction_models.py index ad1d4de..aaaf230 100644 --- a/prediction/prediction_models.py +++ b/prediction/prediction_models.py @@ -17,7 +17,7 @@ def predict_onnx_model( def predict_ov_model( - ov_model: ov.CompiledModel, img_batch: np.ndarray, topk: int, categories: List[str], to_skip: bool = False + ov_model: ov.CompiledModel, img_batch: np.ndarray, topk: int, categories: List[str] ): make_prediction(ov_model, img_batch.cpu().numpy(), topk, categories) From 4fea89eec2b4d0cba9daa8f286d704304fbb788b Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 22:49:31 +0300 Subject: [PATCH 23/52] Fixed imports and refactored --- benchmark/benchmark_utils.py | 6 +++++- main.py | 2 +- src/benchmark_class.py | 3 --- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index 3c7ec0e..38973be 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -49,11 +49,13 @@ def run_all_benchmarks( results = {} # ONNX benchmark + logging.info(f"Running benchmark inference for ONNX model") onnx_benchmark = ONNXBenchmark(models["onnx"], img_batch.shape) avg_time_onnx = onnx_benchmark.run() results["ONNX"] = avg_time_onnx # OpenVINO benchmark + logging.info(f"Running benchmark inference for OpenVINO model") ov_benchmark = OVBenchmark(models["ov"], img_batch.shape) avg_time_ov = ov_benchmark.run() results["OpenVINO"] = avg_time_ov @@ -72,6 +74,7 @@ def run_all_benchmarks( pytorch_benchmark = PyTorchBenchmark( model_to_use, device=device, dtype=precision ) + logging.info(f"Running benchmark inference for PyTorch_{device} model") avg_time_pytorch = pytorch_benchmark.run() results[f"PyTorch_{device}"] = avg_time_pytorch @@ -79,6 +82,7 @@ def run_all_benchmarks( # TensorRT benchmarks if precision == torch.float32 or precision == torch.float16: mode = "fp32" if precision == torch.float32 else "fp16" + logging.info(f"Running benchmark inference for TRT_{mode} model") trt_benchmark = PyTorchBenchmark( models[f"trt_{mode}"], device=device, dtype=precision ) @@ -106,7 +110,7 @@ def plot_benchmark_results(results: Dict[str, float]): # Plot plt.figure(figsize=(10, 6)) - ax = sns.barplot(x=data["Time"], y=data["Model"], palette="rocket") + ax = sns.barplot(x=data["Time"], y=data["Model"], hue=data["Model"], palette="rocket", legend=False) # Adding the actual values on the bars for index, value in enumerate(data["Time"]): diff --git a/main.py b/main.py index 3ccb895..efc7595 100644 --- a/main.py +++ b/main.py @@ -17,7 +17,7 @@ logging.basicConfig(filename="model.log", level=logging.INFO) -def main() -> None: +def main(): """ Main function to run inference, benchmarks, and predictions on the model using provided image and optional parameters. diff --git a/src/benchmark_class.py b/src/benchmark_class.py index 9e866ae..b908425 100644 --- a/src/benchmark_class.py +++ b/src/benchmark_class.py @@ -90,9 +90,6 @@ def run(self): f"Iteration {i}/{self.nruns}, ave batch time {np.mean(timings) * 1000:.2f} ms" ) - # Print and log results - print(f"Input shape: {input_data.size()}") - print(f"Output features size: {features.size()}") logging.info(f"Average batch time: {np.mean(timings) * 1000:.2f} ms") return np.mean(timings) * 1000 From 190c13bf6ac1e3f32fcf86af6e753b604673d9b5 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 23:00:41 +0300 Subject: [PATCH 24/52] Fixed imports and refactored --- README.md | 68 +++++------------------------------------- src/benchmark_class.py | 14 +++++++-- 2 files changed, 19 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 2be52b3..533e9fc 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ - [TensorRT FP32 & FP16](#tensorrt-fp32--fp16) - [ONNX](#onnx) - [OpenVINO](#openvino) -5. [Used methodologies](#used-methodologies) ![New](https://img.shields.io/badge/-New-96E5FE) +5. [Benchmarking and Visualization](#benchmarking-and-visualization) ![New](https://img.shields.io/badge/-New-96E5FE) - [TensorRT Optimization](#tensorrt-optimization) - [ONNX Exporter](#onnx-exporter) - [OV Exporter](#ov-exporter) @@ -44,20 +44,20 @@ docker build -t awesome-tensorrt docker run --gpus all --rm -it awesome-tensorrt # 3. Run the Script inside the Container -python src/main.py +python main.py [--mode all] ``` ### Arguments - `--image_path`: (Optional) Specifies the path to the image you want to predict. - `--topk`: (Optional) Specifies the number of top predictions to show. Defaults to 5 if not provided. -- `--mode`: Specifies the mode for exporting and running the model. Choices are: `onnx`, `ov`, `all`. +- `--mode`: (Optional) Specifies the mode for exporting and running the model. Choices are: `onnx`, `ov`, `all`. If not provided, it defaults to `all`. ### Example Command ```sh -python src/main.py --topk 3 --mode=all +python main.py --topk 3 --mode=ov ``` -This command will run predictions on the default image (`./inference/cat3.jpg`), show the top 3 predictions, and run all models (PyTorch CPU, CUDA, ONNX, OV, TRT-FP16, TRT-FP32). At the end results plot will be saved to `./inference/plot.png` +This command will run predictions on the default image (`./inference/cat3.jpg`), show the top 3 predictions, and run OpenVINO model. At the end results plot will be saved to `./inference/plot.png` ## RESULTS ### Inference Benchmark Results @@ -116,62 +116,8 @@ OpenVINO is a toolkit from Intel that optimizes deep learning model inference fo 4. Perform inference on the provided image using the OpenVINO model. 5. Benchmark results, including average inference time, are logged for the OpenVINO model. -## Used methodologies -### TensorRT Optimization -TensorRT is a high-performance deep learning inference optimizer and runtime library developed by NVIDIA. It is designed for optimizing and deploying trained neural network models on production environments. This project supports TensorRT optimizations in FP32 (single precision) and FP16 (half precision) modes, offering different trade-offs between inference speed and model accuracy. - -#### Features -- **Performance Boost**: TensorRT can significantly accelerate the inference of neural network models, making it suitable for deployment in resource-constrained environments. -- **Precision Modes**: Supports FP32 for maximum accuracy and FP16 for faster performance with a minor trade-off in accuracy. -- **Layer Fusion**: TensorRT fuses layers and tensors in the neural network to reduce memory access overhead and improve execution speed. -- **Dynamic Tensor Memory**: Efficiently handles varying batch sizes without re-optimization. - -#### Usage -When running the main script, use the'- mode all' argument to employ TensorRT optimizations in the project. -This will initiate all models, including PyTorch models, that will be compiled to the TRT model with `FP16` and `FP32` precision modes. Then, in one of the steps, we will run inference on the specified image using the TensorRT-optimized model. -Example: -```sh -python src/main.py --mode all -``` -#### Requirements -Ensure you have the TensorRT library and the torch_tensorrt package installed in your environment. Also, for FP16 optimizations, it's recommended to have a GPU that supports half-precision arithmetic (like NVIDIA GPUs with Tensor Cores). - -### ONNX Exporter -ONNX Model Exporter (`ONNXExporter`) utility is incorporated within this project to enable converting the native PyTorch model into the ONNX format. -Using the ONNX format, inference and benchmarking can be performed with the ONNX Runtime, which offers platform-agnostic optimizations and is widely supported across numerous platforms and devices. - -#### Features -- **Standardized Format**: ONNX provides an open-source format for AI models. It defines an extensible computation graph model and definitions of built-in operators and standard data types. -- **Interoperability**: Models in ONNX format can be used across various frameworks, tools, runtimes, and compilers. -- **Optimizations**: The ONNX Runtime provides performance optimizations for both cloud and edge devices. - -#### Usage -To leverage the `ONNXExporter` and conduct inference using the ONNX Runtime, utilize the `--mode onnx` argument when executing the main script. -This will initiate the conversion process and then run inference on the specified image using the ONNX model. -Example: -```sh -python src/main.py --mode onnx -``` - -#### Requirements -Ensure the ONNX library is installed in your environment to use the ONNXExporter. Additionally, if you want to run inference using the ONNX model, install the ONNX Runtime. - -### OV Exporter -OpenVINO Model Exporter utility (`OVExporter`) has been integrated into this project to facilitate the conversion of the ONNX model to the OpenVINO format. -This enables inference and benchmarking using OpenVINO, a framework optimized for Intel hardware, providing substantial speed improvements, especially on CPUs. - -#### Features -- **Model Optimization**: Converts the ONNX model to OpenVINO's Intermediate Representation (IR) format. This optimized format allows for faster inference times on Intel hardware. -- **Versatility**: OpenVINO can target various Intel hardware devices such as CPUs, integrated GPUs, FPGAs, and VPUs. -- **Ease of Use**: The `OVExporter` seamlessly transitions from ONNX to OpenVINO, abstracting the conversion details and providing a straightforward interface. - -#### Usage -To utilize `OVExporter` and perform inference using OpenVINO, use the `--mode ov` argument when running the main script. -This will trigger the conversion process and subsequently run inference on the provided image using the optimized OpenVINO model. -Example: -```sh -python src/main.py --mode ov -``` +## Benchmarking and Visualization +The results of the benchmarks for all modes are saved and visualized in a bar chart, showcasing the average inference times across different backends. The visualization aids in comparing the performance gains achieved with different optimizations. #### Requirements Ensure you have installed the OpenVINO Toolkit and the necessary dependencies to use OpenVINO's model optimizer and inference engine. diff --git a/src/benchmark_class.py b/src/benchmark_class.py index b908425..6465299 100644 --- a/src/benchmark_class.py +++ b/src/benchmark_class.py @@ -124,12 +124,17 @@ def run(self): print("Starting benchmark ...") timings = [] - for _ in range(self.nruns): + for i in range(1, self.nruns+1): start_time = time.time() _ = self.ort_session.run(None, {"input": input_data}) end_time = time.time() timings.append(end_time - start_time) + if i % 10 == 0: + print( + f"Iteration {i}/{self.nruns}, ave batch time {np.mean(timings) * 1000:.2f} ms" + ) + avg_time = np.mean(timings) * 1000 logging.info(f"Average ONNX inference time: {avg_time:.2f} ms") return avg_time @@ -185,11 +190,16 @@ def run(self): # Benchmarking total_time = 0 - for _ in range(self.num_runs): + for i in range(1, self.num_runs+1): start_time = time.time() _ = self.inference(self.dummy_input) total_time += time.time() - start_time + if i % 10 == 0: + print( + f"Iteration {i}/{self.nruns}, ave batch time {total_time / self.num_runs * 1000:.2f} ms" + ) + avg_time = total_time / self.num_runs logging.info(f"Average inference time: {avg_time * 1000:.2f} ms") return avg_time * 1000 From 5991cddce56d1b60096cb6da7300be834b0ccb25 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 23:04:52 +0300 Subject: [PATCH 25/52] Fixed imports and refactored --- README.md | 2 +- src/benchmark_class.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 533e9fc..d6d8140 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ python main.py [--mode all] python main.py --topk 3 --mode=ov ``` -This command will run predictions on the default image (`./inference/cat3.jpg`), show the top 3 predictions, and run OpenVINO model. At the end results plot will be saved to `./inference/plot.png` +This command will run predictions on the default image (`./inference/cat3.jpg`), show the top 3 predictions, and run OpenVINO model. Note: plot created only for `--mode=all` and results plotted and saved to `./inference/plot.png` ## RESULTS ### Inference Benchmark Results diff --git a/src/benchmark_class.py b/src/benchmark_class.py index 6465299..47e8069 100644 --- a/src/benchmark_class.py +++ b/src/benchmark_class.py @@ -156,8 +156,8 @@ def __init__( self.core = ov.Core() self.compiled_model = None self.input_shape = input_shape - self.warmup_runs = 50 - self.num_runs = 100 + self.nwarmup = 50 + self.nruns = 100 self.dummy_input = np.random.randn(*input_shape).astype(np.float32) def warmup(self): @@ -185,21 +185,21 @@ def run(self): """ # Warm-up runs logging.info("Warming up ...") - for _ in range(self.warmup_runs): + for _ in range(self.nwarmup): self.warmup() # Benchmarking total_time = 0 - for i in range(1, self.num_runs+1): + for i in range(1, self.nruns+1): start_time = time.time() _ = self.inference(self.dummy_input) total_time += time.time() - start_time if i % 10 == 0: print( - f"Iteration {i}/{self.nruns}, ave batch time {total_time / self.num_runs * 1000:.2f} ms" + f"Iteration {i}/{self.nruns}, ave batch time {total_time / self.nruns * 1000:.2f} ms" ) - avg_time = total_time / self.num_runs + avg_time = total_time / self.nruns logging.info(f"Average inference time: {avg_time * 1000:.2f} ms") return avg_time * 1000 From bf4e322cd63cbea4d7a5dfbc2a8d39339e6c5067 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 23:31:43 +0300 Subject: [PATCH 26/52] Fixed imports and refactored --- README.md | 10 ++++++++++ common/utils.py | 11 ++++++++--- main.py | 6 +++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d6d8140..7feb195 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ 3. [RESULTS](#results) ![Static Badge](https://img.shields.io/badge/update-orange) - [Results explanation](#results-explanation) - [Example Input](#example-input) + - [Example prediction results](#example-prediction-results) 4. [Benchmark Implementation Details](#benchmark-implementation-details) ![New](https://img.shields.io/badge/-New-842E5B) - [PyTorch CPU & CUDA](#pytorch-cpu--cuda) - [TensorRT FP32 & FP16](#tensorrt-fp32--fp16) @@ -76,6 +77,15 @@ Here is an example of the input image to run predictions and benchmarks on: +### Example prediction results +``` +#1: 15% Egyptian cat +#2: 14% tiger cat +#3: 9% tabby +#4: 2% doormat +#5: 2% lynx +``` + ## Benchmark Implementation Details Here you can see the flow for each model and benchmark. diff --git a/common/utils.py b/common/utils.py index cd3e120..495d17b 100644 --- a/common/utils.py +++ b/common/utils.py @@ -7,12 +7,17 @@ import onnxruntime as ort -# Model Initialization Functions -def init_onnx_model( +def export_onnx_model( onnx_path: str, model_loader: ModelLoader, device: torch.device -) -> ort.InferenceSession: +) -> None: onnx_exporter = ONNXExporter(model_loader.model, device, onnx_path) onnx_exporter.export_model() + + +def init_onnx_model( + onnx_path: str, model_loader: ModelLoader, device: torch.device +) -> ort.InferenceSession: + export_onnx_model(model_loader, device, onnx_path) return ort.InferenceSession(onnx_path, providers=["CPUExecutionProvider"]) diff --git a/main.py b/main.py index efc7595..9ab5069 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,6 @@ import logging +import os.path + import torch_tensorrt from benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model @@ -7,7 +9,7 @@ parse_arguments, init_onnx_model, init_ov_model, - init_cuda_model, + init_cuda_model, export_onnx_model, ) from src.image_processor import ImageProcessor from prediction.prediction_models import * @@ -42,6 +44,8 @@ def main(): if args.mode in ["ov", "all"]: ov_model = init_ov_model(args.onnx_path) if args.mode != "all": + if not os.path.isfile(args.onnx_path): + export_onnx_model(model_loader, device, args.onnx_path) ov_benchmark = benchmark_ov_model(ov_model) predict_ov_model(ov_benchmark.compiled_model, img_batch, args.topk, model_loader.categories) From 318f0058d6eba2327d69d44b6852c62d610bae62 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 23:33:27 +0300 Subject: [PATCH 27/52] Fixed imports and refactored --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 9ab5069..be90d4b 100644 --- a/main.py +++ b/main.py @@ -42,10 +42,11 @@ def main(): # OpenVINO if args.mode in ["ov", "all"]: + # If ONNX model wasn't exporter previously - export it + if not os.path.isfile(args.onnx_path): + export_onnx_model(model_loader, device, args.onnx_path) ov_model = init_ov_model(args.onnx_path) if args.mode != "all": - if not os.path.isfile(args.onnx_path): - export_onnx_model(model_loader, device, args.onnx_path) ov_benchmark = benchmark_ov_model(ov_model) predict_ov_model(ov_benchmark.compiled_model, img_batch, args.topk, model_loader.categories) From 9da80631a7eaa3202a588f6d42d326c20a287bca Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 23:36:28 +0300 Subject: [PATCH 28/52] Fixed imports and refactored --- common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/utils.py b/common/utils.py index 495d17b..8cadf76 100644 --- a/common/utils.py +++ b/common/utils.py @@ -17,7 +17,7 @@ def export_onnx_model( def init_onnx_model( onnx_path: str, model_loader: ModelLoader, device: torch.device ) -> ort.InferenceSession: - export_onnx_model(model_loader, device, onnx_path) + export_onnx_model(onnx_path=onnx_path, model_loader=model_loader, device=device) return ort.InferenceSession(onnx_path, providers=["CPUExecutionProvider"]) From e0b70180ef8280732c1c9952c7f5fbee2a8ab7b8 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 23:38:50 +0300 Subject: [PATCH 29/52] Fixed imports and refactored --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index be90d4b..7e34521 100644 --- a/main.py +++ b/main.py @@ -42,9 +42,10 @@ def main(): # OpenVINO if args.mode in ["ov", "all"]: - # If ONNX model wasn't exporter previously - export it + # Check if ONNX model wasn't exported previously if not os.path.isfile(args.onnx_path): - export_onnx_model(model_loader, device, args.onnx_path) + export_onnx_model(onnx_path=args.onnx_path, model_loader=model_loader, device=device) + ov_model = init_ov_model(args.onnx_path) if args.mode != "all": ov_benchmark = benchmark_ov_model(ov_model) From 5c0f21ab44624f21627192296efd0e5690550b4f Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Fri, 6 Oct 2023 23:59:26 +0300 Subject: [PATCH 30/52] Fixed logging message to include device type for PyTorch prediction loop. --- prediction/prediction_utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index a9ea429..46d6cd9 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -4,6 +4,7 @@ import torch import onnxruntime as ort import numpy as np +import torch_tensorrt def make_prediction( @@ -16,7 +17,7 @@ def make_prediction( """ Make and print predictions for the given model, img_batch, topk, and categories. - :param model: The model (or ONNX Runtime InferenceSession) to make predictions with. + :param model: The model to make predictions with. :param img_batch: The batch of images to make predictions on. :param topk: The number of top predictions to show. :param categories: The list of categories to label the predictions. @@ -58,7 +59,13 @@ def make_prediction( prob = np.exp(prob[0]) / np.sum(np.exp(prob[0])) else: # PyTorch Model - logging.info(f"Running prediction for PyTorch model") + if isinstance(model, torch.nn.Module): + logging.info(f"Running prediction for PyTorch_{next(model.parameters()).device}") + elif isinstance(model, torch_tensorrt.ts.TSModule): + logging.info(f"Running prediction for TensorRT_{precision} model") + else: + raise ValueError("Running prediction for an unknown model type") + if isinstance(img_batch, np.ndarray): img_batch = torch.tensor(img_batch) else: From 410cbede0bdc019130f6035955871dcb0fcfad90 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 00:08:06 +0300 Subject: [PATCH 31/52] Fixed logging message to include device type for PyTorch prediction loop. --- prediction/prediction_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index 46d6cd9..a3cf940 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -66,6 +66,7 @@ def make_prediction( else: raise ValueError("Running prediction for an unknown model type") + if isinstance(img_batch, np.ndarray): img_batch = torch.tensor(img_batch) else: From 56dee2c78e73747ba907b7f55afcd3db0abf837f Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 00:32:22 +0300 Subject: [PATCH 32/52] Fixed logging message to include device type for PyTorch prediction loop. --- prediction/prediction_utils.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index a3cf940..3507aa7 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -59,13 +59,7 @@ def make_prediction( prob = np.exp(prob[0]) / np.sum(np.exp(prob[0])) else: # PyTorch Model - if isinstance(model, torch.nn.Module): - logging.info(f"Running prediction for PyTorch_{next(model.parameters()).device}") - elif isinstance(model, torch_tensorrt.ts.TSModule): - logging.info(f"Running prediction for TensorRT_{precision} model") - else: - raise ValueError("Running prediction for an unknown model type") - + logging.info(f"Running prediction for PyTorch model") if isinstance(img_batch, np.ndarray): img_batch = torch.tensor(img_batch) From 5c80c97de2c8a0244b1cd70137e7575fbe64fa84 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 00:54:24 +0300 Subject: [PATCH 33/52] Fixed logging message to include device type for PyTorch prediction loop. --- prediction/prediction_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index a3cf940..33ca295 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -59,8 +59,9 @@ def make_prediction( prob = np.exp(prob[0]) / np.sum(np.exp(prob[0])) else: # PyTorch Model - if isinstance(model, torch.nn.Module): - logging.info(f"Running prediction for PyTorch_{next(model.parameters()).device}") + params = list(model.parameters()) + if params: + logging.info(f"Running prediction for PyTorch_{params[0].device}") elif isinstance(model, torch_tensorrt.ts.TSModule): logging.info(f"Running prediction for TensorRT_{precision} model") else: From e06500eabe376897da3edf76fe2847c79ed8b890 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 00:58:46 +0300 Subject: [PATCH 34/52] Fixed logging message to include device type for PyTorch prediction loop. --- prediction/prediction_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index 33ca295..a34366f 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -62,12 +62,11 @@ def make_prediction( params = list(model.parameters()) if params: logging.info(f"Running prediction for PyTorch_{params[0].device}") - elif isinstance(model, torch_tensorrt.ts.TSModule): + elif isinstance(model, torch.nn.Module): logging.info(f"Running prediction for TensorRT_{precision} model") else: raise ValueError("Running prediction for an unknown model type") - if isinstance(img_batch, np.ndarray): img_batch = torch.tensor(img_batch) else: From f3bb8cb1d118ab248a7020373db2dedd28029f55 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 01:08:32 +0300 Subject: [PATCH 35/52] IMG Batch size fix in TRT models --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 7e34521..ea037b6 100644 --- a/main.py +++ b/main.py @@ -77,7 +77,7 @@ def main(): print("Compiling TensorRT model") model = torch_tensorrt.compile( model, - inputs=[torch_tensorrt.Input((32, 3, 224, 224), dtype=precision)], + inputs=[torch_tensorrt.Input((1, 3, 224, 224), dtype=precision)], enabled_precisions={precision}, truncate_long_and_double=True, ) From 4bf4485df4c8ef2fdc1471244db6020574e634b2 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 01:13:36 +0300 Subject: [PATCH 36/52] Fixed warnings --- benchmark/benchmark_utils.py | 8 +++++++- main.py | 23 ++++++++++++++++++----- src/benchmark_class.py | 4 ++-- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index 38973be..3b44513 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -110,7 +110,13 @@ def plot_benchmark_results(results: Dict[str, float]): # Plot plt.figure(figsize=(10, 6)) - ax = sns.barplot(x=data["Time"], y=data["Model"], hue=data["Model"], palette="rocket", legend=False) + ax = sns.barplot( + x=data["Time"], + y=data["Model"], + hue=data["Model"], + palette="rocket", + legend=False, + ) # Adding the actual values on the bars for index, value in enumerate(data["Time"]): diff --git a/main.py b/main.py index ea037b6..df74c4f 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,5 @@ import logging import os.path - import torch_tensorrt from benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model @@ -9,11 +8,15 @@ parse_arguments, init_onnx_model, init_ov_model, - init_cuda_model, export_onnx_model, + init_cuda_model, + export_onnx_model, ) from src.image_processor import ImageProcessor from prediction.prediction_models import * from src.model import ModelLoader +import os + +os.environ["CUDA_LAZY_DEBUG"] = "1" # Configure logging logging.basicConfig(filename="model.log", level=logging.INFO) @@ -38,18 +41,27 @@ def main(): ort_session = init_onnx_model(args.onnx_path, model_loader, device) if args.mode != "all": benchmark_onnx_model(ort_session) - predict_onnx_model(ort_session, img_batch, args.topk, model_loader.categories) + predict_onnx_model( + ort_session, img_batch, args.topk, model_loader.categories + ) # OpenVINO if args.mode in ["ov", "all"]: # Check if ONNX model wasn't exported previously if not os.path.isfile(args.onnx_path): - export_onnx_model(onnx_path=args.onnx_path, model_loader=model_loader, device=device) + export_onnx_model( + onnx_path=args.onnx_path, model_loader=model_loader, device=device + ) ov_model = init_ov_model(args.onnx_path) if args.mode != "all": ov_benchmark = benchmark_ov_model(ov_model) - predict_ov_model(ov_benchmark.compiled_model, img_batch, args.topk, model_loader.categories) + predict_ov_model( + ov_benchmark.compiled_model, + img_batch, + args.topk, + model_loader.categories, + ) # CUDA if args.mode in ["cuda", "all"]: @@ -80,6 +92,7 @@ def main(): inputs=[torch_tensorrt.Input((1, 3, 224, 224), dtype=precision)], enabled_precisions={precision}, truncate_long_and_double=True, + require_full_compilation=True, ) # If it is for TensorRT, determine the mode (FP32 or FP16) and store under a TensorRT key mode = "fp32" if precision == torch.float32 else "fp16" diff --git a/src/benchmark_class.py b/src/benchmark_class.py index 47e8069..b1800cf 100644 --- a/src/benchmark_class.py +++ b/src/benchmark_class.py @@ -124,7 +124,7 @@ def run(self): print("Starting benchmark ...") timings = [] - for i in range(1, self.nruns+1): + for i in range(1, self.nruns + 1): start_time = time.time() _ = self.ort_session.run(None, {"input": input_data}) end_time = time.time() @@ -190,7 +190,7 @@ def run(self): # Benchmarking total_time = 0 - for i in range(1, self.nruns+1): + for i in range(1, self.nruns + 1): start_time = time.time() _ = self.inference(self.dummy_input) total_time += time.time() - start_time From 6ae5e8457ad0eceb8629323ab6cf5302d97a93bf Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 01:15:51 +0300 Subject: [PATCH 37/52] Fixed warnings --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index df74c4f..b85f6f5 100644 --- a/main.py +++ b/main.py @@ -14,9 +14,10 @@ from src.image_processor import ImageProcessor from prediction.prediction_models import * from src.model import ModelLoader -import os +import warnings -os.environ["CUDA_LAZY_DEBUG"] = "1" +# Filter out the specific warning from torchvision +warnings.filterwarnings("ignore", category=UserWarning, module="torchvision.io.image") # Configure logging logging.basicConfig(filename="model.log", level=logging.INFO) From ac3dc533e2a6259ed39a3e74d4260fe808501ab3 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 01:25:53 +0300 Subject: [PATCH 38/52] Fixed warnings --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index b85f6f5..058d9f9 100644 --- a/main.py +++ b/main.py @@ -88,9 +88,10 @@ def main(): img_batch = img_batch.to(device) else: print("Compiling TensorRT model") + batch_size = 1 if args.mode == "cuda" else 32 model = torch_tensorrt.compile( model, - inputs=[torch_tensorrt.Input((1, 3, 224, 224), dtype=precision)], + inputs=[torch_tensorrt.Input((batch_size, 3, 224, 224), dtype=precision)], enabled_precisions={precision}, truncate_long_and_double=True, require_full_compilation=True, From e92988197c02e297677836d00be3e36668db391a Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 15:21:09 +0300 Subject: [PATCH 39/52] Delete benchmark_class.py --- src/benchmark_class.py | 205 ----------------------------------------- 1 file changed, 205 deletions(-) delete mode 100644 src/benchmark_class.py diff --git a/src/benchmark_class.py b/src/benchmark_class.py deleted file mode 100644 index b1800cf..0000000 --- a/src/benchmark_class.py +++ /dev/null @@ -1,205 +0,0 @@ -import time -from typing import Tuple - -from abc import ABC, abstractmethod -import numpy as np -import torch -import torch.backends.cudnn as cudnn -import logging -import onnxruntime as ort -import openvino as ov - -# Configure logging -logging.basicConfig(filename="model.log", level=logging.INFO) - - -class Benchmark(ABC): - """ - Abstract class representing a benchmark. - """ - - def __init__(self, nruns: int = 100, nwarmup: int = 50): - self.nruns = nruns - self.nwarmup = nwarmup - - @abstractmethod - def run(self): - """ - Abstract method to run the benchmark. - """ - pass - - -class PyTorchBenchmark: - def __init__( - self, - model: torch.nn.Module, - device: str = "cuda", - input_shape: Tuple[int, int, int, int] = (32, 3, 224, 224), - dtype: torch.dtype = torch.float32, - nwarmup: int = 50, - nruns: int = 100, - ) -> None: - """ - Initialize the Benchmark object. - - :param model: The model to be benchmarked. - :param device: The device to run the benchmark on ("cpu" or "cuda"). - :param input_shape: The shape of the input data. - :param dtype: The data type to be used in the benchmark (typically torch.float32 or torch.float16). - :param nwarmup: The number of warmup runs before timing. - :param nruns: The number of runs for timing. - """ - self.model = model - self.device = device - self.input_shape = input_shape - self.dtype = dtype - self.nwarmup = nwarmup - self.nruns = nruns - - cudnn.benchmark = True # Enable cuDNN benchmarking optimization - - def run(self): - """ - Run the benchmark with the given model, input shape, and other parameters. - Log the average batch time and print the input shape and output feature size. - """ - # Prepare input data - input_data = torch.randn(self.input_shape).to(self.device).to(self.dtype) - - # Warm up - print("Warm up ...") - with torch.no_grad(): - for _ in range(self.nwarmup): - features = self.model(input_data) - torch.cuda.synchronize() - - # Start timing - print("Start timing ...") - timings = [] - with torch.no_grad(): - for i in range(1, self.nruns + 1): - start_time = time.time() - features = self.model(input_data) - torch.cuda.synchronize() - end_time = time.time() - timings.append(end_time - start_time) - - if i % 10 == 0: - print( - f"Iteration {i}/{self.nruns}, ave batch time {np.mean(timings) * 1000:.2f} ms" - ) - - logging.info(f"Average batch time: {np.mean(timings) * 1000:.2f} ms") - return np.mean(timings) * 1000 - - -class ONNXBenchmark(Benchmark): - """ - A class used to benchmark the performance of an ONNX model. - """ - - def __init__( - self, - ort_session: ort.InferenceSession, - input_shape: tuple, - nruns: int = 100, - nwarmup: int = 50, - ): - super().__init__(nruns) - self.ort_session = ort_session - self.input_shape = input_shape - self.nwarmup = nwarmup - self.nruns = nruns - - def run(self): - print("Warming up ...") - # Adjusting the batch size in the input shape to match the expected input size of the model. - input_shape = (1,) + self.input_shape[1:] - input_data = np.random.randn(*input_shape).astype(np.float32) - - for _ in range(self.nwarmup): # Warm-up runs - _ = self.ort_session.run(None, {"input": input_data}) - - print("Starting benchmark ...") - timings = [] - - for i in range(1, self.nruns + 1): - start_time = time.time() - _ = self.ort_session.run(None, {"input": input_data}) - end_time = time.time() - timings.append(end_time - start_time) - - if i % 10 == 0: - print( - f"Iteration {i}/{self.nruns}, ave batch time {np.mean(timings) * 1000:.2f} ms" - ) - - avg_time = np.mean(timings) * 1000 - logging.info(f"Average ONNX inference time: {avg_time:.2f} ms") - return avg_time - - -class OVBenchmark(Benchmark): - def __init__( - self, model: ov.frontend.FrontEnd, input_shape: Tuple[int, int, int, int] - ): - """ - Initialize the OVBenchmark with the OpenVINO model and the input shape. - - :param model: ov.frontend.FrontEnd - The OpenVINO model. - :param input_shape: Tuple[int, int, int, int] - The shape of the model input. - """ - self.ov_model = model - self.core = ov.Core() - self.compiled_model = None - self.input_shape = input_shape - self.nwarmup = 50 - self.nruns = 100 - self.dummy_input = np.random.randn(*input_shape).astype(np.float32) - - def warmup(self): - """ - Compile the OpenVINO model for optimal execution on available hardware. - """ - self.compiled_model = self.core.compile_model(self.ov_model, "AUTO") - - def inference(self, input_data) -> dict: - """ - Perform inference on the input data using the compiled OpenVINO model. - - :param input_data: np.ndarray - The input data for the model. - :return: dict - The model's output as a dictionary. - """ - outputs = self.compiled_model(inputs={"input": input_data}) - return outputs - - def run(self): - """ - Run the benchmark on the OpenVINO model. It first warms up by compiling the model and then measures - the average inference time over a set number of runs. - """ - # Warm-up runs - logging.info("Warming up ...") - for _ in range(self.nwarmup): - self.warmup() - - # Benchmarking - total_time = 0 - for i in range(1, self.nruns + 1): - start_time = time.time() - _ = self.inference(self.dummy_input) - total_time += time.time() - start_time - - if i % 10 == 0: - print( - f"Iteration {i}/{self.nruns}, ave batch time {total_time / self.nruns * 1000:.2f} ms" - ) - - avg_time = total_time / self.nruns - logging.info(f"Average inference time: {avg_time * 1000:.2f} ms") - return avg_time * 1000 From 93b6a4597b1edb30f1bcecfd5225f1d77d2873d4 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 15:30:38 +0300 Subject: [PATCH 40/52] Refactor benchmark --- benchmark/benchmark_models.py | 208 +++++++++++++++++++++++++++++++++- benchmark/benchmark_utils.py | 2 +- 2 files changed, 205 insertions(+), 5 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index e772360..3f2a549 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -1,16 +1,216 @@ -import src.benchmark_class from benchmark.benchmark_utils import run_benchmark -from src.benchmark_class import PyTorchBenchmark, ONNXBenchmark, OVBenchmark -import openvino as ov +import time +from typing import Tuple + +from abc import ABC, abstractmethod +import numpy as np import torch +import torch.backends.cudnn as cudnn +import logging import onnxruntime as ort +import openvino as ov + +# Configure logging +logging.basicConfig(filename="model.log", level=logging.INFO) + + +class Benchmark(ABC): + """ + Abstract class representing a benchmark. + """ + + def __init__(self, nruns: int = 100, nwarmup: int = 50): + self.nruns = nruns + self.nwarmup = nwarmup + + @abstractmethod + def run(self): + """ + Abstract method to run the benchmark. + """ + pass + + +class PyTorchBenchmark: + def __init__( + self, + model: torch.nn.Module, + device: str = "cuda", + input_shape: Tuple[int, int, int, int] = (32, 3, 224, 224), + dtype: torch.dtype = torch.float32, + nwarmup: int = 50, + nruns: int = 100, + ) -> None: + """ + Initialize the Benchmark object. + + :param model: The model to be benchmarked. + :param device: The device to run the benchmark on ("cpu" or "cuda"). + :param input_shape: The shape of the input data. + :param dtype: The data type to be used in the benchmark (typically torch.float32 or torch.float16). + :param nwarmup: The number of warmup runs before timing. + :param nruns: The number of runs for timing. + """ + self.model = model + self.device = device + self.input_shape = input_shape + self.dtype = dtype + self.nwarmup = nwarmup + self.nruns = nruns + + cudnn.benchmark = True # Enable cuDNN benchmarking optimization + + def run(self): + """ + Run the benchmark with the given model, input shape, and other parameters. + Log the average batch time and print the input shape and output feature size. + """ + # Prepare input data + input_data = torch.randn(self.input_shape).to(self.device).to(self.dtype) + + # Warm up + print("Warm up ...") + with torch.no_grad(): + for _ in range(self.nwarmup): + features = self.model(input_data) + torch.cuda.synchronize() + + # Start timing + print("Start timing ...") + timings = [] + with torch.no_grad(): + for i in range(1, self.nruns + 1): + start_time = time.time() + features = self.model(input_data) + torch.cuda.synchronize() + end_time = time.time() + timings.append(end_time - start_time) + + if i % 10 == 0: + print( + f"Iteration {i}/{self.nruns}, ave batch time {np.mean(timings) * 1000:.2f} ms" + ) + + logging.info(f"Average batch time: {np.mean(timings) * 1000:.2f} ms") + return np.mean(timings) * 1000 + + +class ONNXBenchmark(Benchmark): + """ + A class used to benchmark the performance of an ONNX model. + """ + + def __init__( + self, + ort_session: ort.InferenceSession, + input_shape: tuple, + nruns: int = 100, + nwarmup: int = 50, + ): + super().__init__(nruns) + self.ort_session = ort_session + self.input_shape = input_shape + self.nwarmup = nwarmup + self.nruns = nruns + + def run(self): + print("Warming up ...") + # Adjusting the batch size in the input shape to match the expected input size of the model. + input_shape = (1,) + self.input_shape[1:] + input_data = np.random.randn(*input_shape).astype(np.float32) + + for _ in range(self.nwarmup): # Warm-up runs + _ = self.ort_session.run(None, {"input": input_data}) + + print("Starting benchmark ...") + timings = [] + + for i in range(1, self.nruns + 1): + start_time = time.time() + _ = self.ort_session.run(None, {"input": input_data}) + end_time = time.time() + timings.append(end_time - start_time) + + if i % 10 == 0: + print( + f"Iteration {i}/{self.nruns}, ave batch time {np.mean(timings) * 1000:.2f} ms" + ) + + avg_time = np.mean(timings) * 1000 + logging.info(f"Average ONNX inference time: {avg_time:.2f} ms") + return avg_time + + +class OVBenchmark(Benchmark): + def __init__( + self, model: ov.frontend.FrontEnd, input_shape: Tuple[int, int, int, int] + ): + """ + Initialize the OVBenchmark with the OpenVINO model and the input shape. + + :param model: ov.frontend.FrontEnd + The OpenVINO model. + :param input_shape: Tuple[int, int, int, int] + The shape of the model input. + """ + self.ov_model = model + self.core = ov.Core() + self.compiled_model = None + self.input_shape = input_shape + self.nwarmup = 50 + self.nruns = 100 + self.dummy_input = np.random.randn(*input_shape).astype(np.float32) + + def warmup(self): + """ + Compile the OpenVINO model for optimal execution on available hardware. + """ + self.compiled_model = self.core.compile_model(self.ov_model, "AUTO") + + def inference(self, input_data) -> dict: + """ + Perform inference on the input data using the compiled OpenVINO model. + + :param input_data: np.ndarray + The input data for the model. + :return: dict + The model's output as a dictionary. + """ + outputs = self.compiled_model(inputs={"input": input_data}) + return outputs + + def run(self): + """ + Run the benchmark on the OpenVINO model. It first warms up by compiling the model and then measures + the average inference time over a set number of runs. + """ + # Warm-up runs + logging.info("Warming up ...") + for _ in range(self.nwarmup): + self.warmup() + + # Benchmarking + total_time = 0 + for i in range(1, self.nruns + 1): + start_time = time.time() + _ = self.inference(self.dummy_input) + total_time += time.time() - start_time + + if i % 10 == 0: + print( + f"Iteration {i}/{self.nruns}, ave batch time {total_time / self.nruns * 1000:.2f} ms" + ) + + avg_time = total_time / self.nruns + logging.info(f"Average inference time: {avg_time * 1000:.2f} ms") + return avg_time * 1000 def benchmark_onnx_model(ort_session: ort.InferenceSession): run_benchmark(None, None, None, ort_session, onnx=True) -def benchmark_ov_model(ov_model: ov.CompiledModel) -> src.benchmark_class.OVBenchmark: +def benchmark_ov_model(ov_model: ov.CompiledModel) -> OVBenchmark: ov_benchmark = OVBenchmark(ov_model, input_shape=(1, 3, 224, 224)) ov_benchmark.run() return ov_benchmark diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index 3b44513..7b61f6d 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -8,7 +8,7 @@ import torch import onnxruntime as ort -from src.benchmark_class import PyTorchBenchmark, ONNXBenchmark, OVBenchmark +from benchmark.benchmark_models import PyTorchBenchmark, ONNXBenchmark, OVBenchmark def run_benchmark( From 834c8bb9c1f41e2b0d6036db91b79ab6b1b78cee Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sat, 7 Oct 2023 16:35:39 +0300 Subject: [PATCH 41/52] Refactor benchmark --- benchmark/benchmark_models.py | 26 +++++++++++++++++++++++++- benchmark/benchmark_utils.py | 24 ------------------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index 3f2a549..b163680 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -1,4 +1,3 @@ -from benchmark.benchmark_utils import run_benchmark import time from typing import Tuple @@ -218,3 +217,28 @@ def benchmark_ov_model(ov_model: ov.CompiledModel) -> OVBenchmark: def benchmark_cuda_model(cuda_model: torch.nn.Module, device: str, dtype: torch.dtype): run_benchmark(cuda_model, device, dtype) + + +def run_benchmark( + model: torch.nn.Module, + device: str, + dtype: torch.dtype, + ort_session: ort.InferenceSession = None, + onnx: bool = False, +) -> None: + """ + Run and log the benchmark for the given model, device, and dtype. + + :param onnx: + :param ort_session: + :param model: The model to be benchmarked. + :param device: The device to run the benchmark on ("cpu" or "cuda"). + :param dtype: The data type to be used in the benchmark (typically torch.float32 or torch.float16). + """ + if onnx: + logging.info(f"Running Benchmark for ONNX") + benchmark = ONNXBenchmark(ort_session, input_shape=(32, 3, 224, 224)) + else: + logging.info(f"Running Benchmark for {device.upper()} and precision {dtype}") + benchmark = PyTorchBenchmark(model, device=device, dtype=dtype) + benchmark.run() \ No newline at end of file diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index 7b61f6d..6636c12 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -11,30 +11,6 @@ from benchmark.benchmark_models import PyTorchBenchmark, ONNXBenchmark, OVBenchmark -def run_benchmark( - model: torch.nn.Module, - device: str, - dtype: torch.dtype, - ort_session: ort.InferenceSession = None, - onnx: bool = False, -) -> None: - """ - Run and log the benchmark for the given model, device, and dtype. - - :param onnx: - :param ort_session: - :param model: The model to be benchmarked. - :param device: The device to run the benchmark on ("cpu" or "cuda"). - :param dtype: The data type to be used in the benchmark (typically torch.float32 or torch.float16). - """ - if onnx: - logging.info(f"Running Benchmark for ONNX") - benchmark = ONNXBenchmark(ort_session, input_shape=(32, 3, 224, 224)) - else: - logging.info(f"Running Benchmark for {device.upper()} and precision {dtype}") - benchmark = PyTorchBenchmark(model, device=device, dtype=dtype) - benchmark.run() - def run_all_benchmarks( models: Dict[str, Any], img_batch: np.ndarray From 29368fc4401fdbfc683895910e7484fa6f82dc49 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Mon, 9 Oct 2023 11:42:25 +0300 Subject: [PATCH 42/52] Fixed average calculation --- benchmark/benchmark_models.py | 2 +- benchmark/benchmark_utils.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index b163680..6a73677 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -197,7 +197,7 @@ def run(self): if i % 10 == 0: print( - f"Iteration {i}/{self.nruns}, ave batch time {total_time / self.nruns * 1000:.2f} ms" + f"Iteration {i}/{self.nruns}, ave batch time {total_time / i * 1000:.2f} ms" ) avg_time = total_time / self.nruns diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index 6636c12..532a4b1 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -11,7 +11,6 @@ from benchmark.benchmark_models import PyTorchBenchmark, ONNXBenchmark, OVBenchmark - def run_all_benchmarks( models: Dict[str, Any], img_batch: np.ndarray ) -> Dict[str, float]: From a42229a8c19ae2265fda16e9f4fc214f798efec0 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Mon, 9 Oct 2023 12:35:50 +0300 Subject: [PATCH 43/52] Updated README.md and results --- README.md | 16 ++++++++-------- inference/plot_latest.png | Bin 0 -> 32759 bytes 2 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 inference/plot_latest.png diff --git a/README.md b/README.md index 4355b24..1fb9d09 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ 8. [References](#references) - + ## Overview This project demonstrates how to perform inference with a PyTorch model and optimize it using ONNX, OpenVINO, and NVIDIA TensorRT. The script loads a pre-trained ResNet-50 model from torch-vision, performs inference on a user-provided image, and prints the top-K predicted classes. Additionally, the script benchmarks the model's performance in the following configurations: PyTorch CPU, ONNX CPU, OpenVINO CPU, PyTorch CUDA, TensorRT-FP32, and TensorRT-FP16, providing insights into the speedup gained through optimization. @@ -60,15 +60,15 @@ This command will run predictions on the chosen image (`./inference/train.jpg`), ## RESULTS ### Inference Benchmark Results - + ### Results explanation - - `PyTorch_cpu: 973.52 ms` indicates the average batch time when running the `PyTorch` model on `CPU` device. - - `PyTorch_cuda: 41.11 ms` indicates the average batch time when running the `PyTorch` model on the `CUDA` device. - - `TRT_fp32: 19.10 ms` shows the average batch time when running the model with `TensorRT` using `float32` precision. - - `TRT_fp16: 7.22 ms` indicates the average batch time when running the model with `TensorRT` using `float16` precision. - - ![New](https://img.shields.io/badge/-New-842E5B)`ONNX: 15.38 ms` indicates the average batch inference time when running the `PyTorch` converted to the `ONNX` model on the `CPU` device. - - ![New](https://img.shields.io/badge/-New-842E5B)`OpenVINO: 14.04 ms` indicates the average batch inference time when running the `ONNX` model converted to `OpenVINO` on the `CPU` device. + - `PyTorch_cpu: 978.71 ms` indicates the average batch time when running the `PyTorch` model on `CPU` device. + - `PyTorch_cuda: 30.11 ms` indicates the average batch time when running the `PyTorch` model on the `CUDA` device. + - `TRT_fp32: 19.20 ms` shows the average batch time when running the model with `TensorRT` using `float32` precision. + - `TRT_fp16: 7.32 ms` indicates the average batch time when running the model with `TensorRT` using `float16` precision. + - ![New](https://img.shields.io/badge/-New-842E5B)`ONNX: 15.95 ms` indicates the average batch inference time when running the `PyTorch` converted to the `ONNX` model on the `CPU` device. + - ![New](https://img.shields.io/badge/-New-842E5B)`OpenVINO: 13.37 ms` indicates the average batch inference time when running the `ONNX` model converted to `OpenVINO` on the `CPU` device. ### Example Input Here is an example of the input image to run predictions and benchmarks on: diff --git a/inference/plot_latest.png b/inference/plot_latest.png new file mode 100644 index 0000000000000000000000000000000000000000..d64d72766dbfa3f49ce6a90b5668f45c940329c7 GIT binary patch literal 32759 zcmdSC2UwNowk=GmiH#U5DqsPXE=@%cVgW=Hlqy}NgMiXI#)45$a3RtJRFEnly@N^+ z5s(f70xC!^N)hRIOw2iZpZ}h+PkHXWzt7H&fwjK%eQ%j_%rVA%*R|scvdb2)U(CeB zv`p^sAtffJ?{b)!zTNuMkN64SdSm>6Nz7LIq^+`*k*&jNYeS}Er)|%jwX!{HdS;Wo zp|y>vm8CHM9{!!Yn@ntN&)Mu15U}{?7w}tI8w-dgr^n+WKc73SVZ+44e476GCPpgS zl!?j1KMUqHn|`2k|xFr zbQ{IgMJw}?P75t9`UN+^#H99YTlA8Jzir}KVf)R(kFMNX_{n4L?{(K{ztYHf^`tIa zxx=!D4qZOF&Ch=5O;GJagTsx9Iv)lG?$#O_c=>Htd}$!WwPnkeX!&nWJ;cR*4>CM@ z72KPeo8wL8@M8Z7<{E#fzOi zJu#-WQKoHqPS)1eFW)Z?XDuyJ9VvVMTt@o(suNyo_|hy|GVbyZ>vcaJ-s&*)W@Kcf zCd8^>vTkmFSAd@TiIXQ!Dg=D<BS%F)6E-I0vrZjpAaaVNs-p72o~jQ zn;&n~i&IYud-LXvi2cCH;^Jb{_kUh%F7znOuxjTQ7tiI5pEy78I{9E@l3t@MyIbUi z3m5M4cgL<^W8*hAHs<8y>@5!xo|@`UQccp`XVG4uqZ+SP(K|9{$0+pq^2IShCs$25 zTA_ZZH8ACvfB$`CVj_TJe4t7Bhb3$3O4vOf zeJBgy*|B3sONLgunX21J>Gr~_6E1U;-Sycv-L})-tm0~)K7DFVHxD>+ zd&g$ExDyec>vvl|!hnqT)%VNYo;r@9@a6bW5wg4NxwnsxN0;%HmUQ#zi$5$0ZqKxA z)%539t-;Lxl*4k{zvSs`yX4;2XU~iFcD=Fyfy^G8i6NaE>qU=9N=jOvJ9qr|Z+}pU zR$!Huk%@Z!>W>v$$0pi6cqAld?u%mVOkdvS!peJqSyRb6cY1uNRZBflTX5nN=KuHa z7Nwmo`f;?Ev24A_VL>%l*Sw)O*?wIacX%}(tl`rPJebJw!}s6EoNY>(!g^rS?C&@% zS?rA23l`M(#D%xIxVT_abj0lkc21aA-TVE(fdd?J_3`Qfr3NX1#d94#Qgx3sgtN1= zxfH`B;`9m&+~;QT5d>$(hz zhKkudKjZ!5vh^`a50C71{CIEuuCp-`P9rhcNx?f$AGq$8i{<(3*)x@lvlzn53NGC=?s~h8@a5kteRR{%4%zsi#(Ur4z^_SO$8m@ zwrt(b-=t>8{J2%)IOu^MojieUe7UWq0u>`{~aX z`Lmx}tQ{Nz{QUfM-Nx%zML*vj-caB^>pVFg!c;S5pJ2l|s4GOmotK)jwQ_E-&@l&M7xJ z@$&f2XHjUC-ZkIH2o%$Maa@$}rjp zXY}>cZMqc(+w!$;Y}lRgTA?+^ZoEcb!1USWpZi)et#VRx@?9p>5H1l%qIS2tDsNY~ zTeVit_5MDW&plYC>R4~0hA&qw+qmaA!khE_>;$LWl_hI4)*08uoLIltHU@DP<0fKM zzE0G%=FhyDfu_`)bMG!Xc}}~Fn$=Yy&M>l>M_0JvojLBaInEP9dqz zb#L#waF03D#C==aM~k!zTs3O)t6V=1Xt<44%VKFf-Xp%5i%TvkZ{+$etF}kF`OdKi zt}Ld|xlJ+bQPEZA{^)S`lyx?vT@{)+wz0V4_O)x*x*>R(wPYkHgi9uT`@`bHj8~!J z(d$Lcdiv_)@xK6Uxg;!Yvrq4fbIx~o7IO`Mw`jRaf$OA*b?4zw2`44xsH4>vzWFZg z+`9u_gKYbpRgwzlXk-&`A#?10v!;}XQHtS-^E2I26f`+Z-(;Pqr@nvis)4?Z)$+|k zLP{k*?CJ=Ao$ub=$K_3)d$a64_x1vEiP4JpHG7w{rRkR}7qw_snVXx_%sAV$HT2r8 z=0X9MeMZw`y%EJ;Yt5fr_{MoCyBnK?lTuH0b+t1dOjxIEmTtwS`xO^H1y7!Oat#Yh z>`hj2tVo%uFC9Ldt+_YnG3AZ-_FrN%&t-ZP_zlyG6?o!Z94zJyj|BO3^Gxm{w4BUy zOnx^sWcj+KWrOjnyMhxIh{W&u`_13v*phtIevPOjL>oSZ=| zR&TOS-{w|*Y4Rp38Lx8bd7T^~eQdj8n27alxm$%F)3s7cX5}wqe5tui=iTZb#={ef#aV zyanNzZO&yc{CNV-kN2tCXq7TJm&rZ9!>3hl0f2=`LJD|}HZuN-{Z@nq-HtsESGp0l{ytrPyY7Xg*UCOA9{OZ?FXAF zUSew1v8^~;LpDz%pRHQ8Dzd*ZsULg18dDa55Fd-kyw|$(k4gRtfR%crVB5|Yx0YJu zVso3letdu1(ZG8lyRD*{jjJC>Tie(uBg9v<%yox*bar=d2Odh$${f1i!gb=oWwj*T z1WC847{jtV6#738G;MWoaPSH_&DnU$$}0SfcjpG1hpNVEeO31_6??0X^oC(iDmvsN zy zQUmg=r|ESArifg;nmfjIa%4@5bMM0d4zA)1EQ#{eR}7VGYs2iEoZtyTL_A8zhzdb{ zO-#(M!an!ampw=TE;@4bXdM;?@6^7DH#Tc_%e%b0vX((rWB4p$iMsm2MOBQ zN=_r4w{B*3Y47rcre|p{s<;Z_2ChJf<<<{HPZe( z>Iyc~r%#_Ia6~Z`TZ5-~WUK7W9zafRm6+qfE$*u(N4qJOinz}>&SG4-b@Q^sHcB{# zM%fDR9zHT*TY-9jTTrlirmd-I@>bYEXOBN_%?*oGUe$>Yo>}DC+3%oigS+(lMF*IE zmHs-<&Ft*qDvcf=$KD}PMgL@t{9jw(HeEjK!zvzyAn`Cu;q%i$qv8o|KyI&)6MzK@ znE&;A&fUXGH*8b0wN1my7fwr9RLI=hCb`XneXW45%GtAz0HSW+UoSj0EBU&zqT(2% zv$r?MYtOaLxQO!>+*4ou@AXVB(5gKs5O?BV~)1FINM;5>{0K7 zSH=+cALh7gp61-j_^t@CS~bt{{Awl+oe$b_x5cLJw3f+!6IDsc5F)-8tFbktO5z{EE-Zfb{y8qI9pwH|DY-g zoKcT5TBL&U0uN^wZ3N7v@uMxKmZ0h1+b!8XfAuQ%C;^txy>>N#rw01^ezE$ya2dP) zhI${Vxfnf}GJ)A#G2nQXV0`LQJ;7PY?*Trj&P7D?Ip2w`cxV5!DL2LKy$0BCu z!SqHUxAHYu9+ZCz-DhpzJ^NuF!q$3WBX6;uuCf3{^(5V6$!!RCY*X#}&LtN^LqjoN zv6v)oL`b}?R$b5S;k%-%A@fy>m?k>b>o7UK6fvp#jd1a8E@j*P2ENtYDlax3P!8$M zn@`@sESd9XQhJ8l z-6xQZ_etkCT>YStY7~pu&=sUDNAQwh%cmWo*vfr~bY|@ZZaLE@a_ucpF$U@>?(LbH zoTLE^-gEv;oR7QQppD~kV20Z~>ino(xr>)ZoLgpPmaavzL0$aHssZn9vhr?&@=yVLj zn4kU45FVb*R03!Oi&O>Gs5(GXy4&<;*~A181r$9ehl+tTSFT*CeAikh=J@8-e42Id z+XI9QU*dKw*7(>ue=O`yGBDseaq@=g(cmo)^toeD`s<*qfh8;)>^T);N?1 zJX^LLaGM&dPBX2g8hmnUs_(|`w&Ms&<3OP1{S67=Eeg{dhIQp8EW>K=?f-6aa+1qq zZl*cgCVKMxCT3w^Zqc)^cP;?2`dkMq6jBN&PMk2SiM&Ro3o?NTkN{O}b%{FSaue>R z%bA%Qk!3y%4kn-)Ko-@ld3?IacHp%jX8y3W*P41nt8tWTTnO34sXOOK!{_N51TRob z)8Zz&168Riwpz;D+6u{CyLL@=g;=dxw=U-TYMwZ)OiRjHV5;J=_e)VY0=?Me=wo1I zB4mBkOiW@F!X#ob`ent%(p1w?cmRT-I2shkvuDpq;PZNvSP3Yy}}5Lht#S<5Tr z+QVWH8jLZFEEVK0fq045AQCD-K(9ewAU_i6w;zxuc&}YtoLWL0z>&d}Fw;kYTPri2 ze`JcBa||}PcI}!>kdUD_7H1%PESNIEt$X&o{`m3ZPOJ}$RXl3tnA|JLxPeh;OwKOw zNvDBU8qe9#gmIY+HewWKivRk9JoY4_vCaoW>?FR=PbrvWKU19crgZH|O?tl+50{ESlH- zuz1x@Fe^7uKCQ>dJqP5eaI2YS1DFQ|bp``dLfHQ9-Mii+AIdh&PEYJ!&brrN2m#YA z;!N>%u7+hl|15aq4zFpoeaG`<9jHTgAMxil?0gm4LIZ^B8E+J~FMpsXQ_x)(`Pp&v z_U+pvP~JRemYVT>Fdr)J;Ek308idk=)4+e7WdW5RM@NN$6AZO;?JM3WD#iJv$qfqb zqWTS)tiq>Ql*jb3<(LT zE^u?{c$00zbu(h@=A~(Nf!hPfAcA2oLt;hvvj+j-x}A{F5CZAa-Me>tmCQ;E1m=UW z6143(9)ZX4^z!n0*V$#yFZ zDAprUc>+2}nVDeO%J^+N=8ZtX0^}fb?N?6kwtSZjf`WpkLsCqEOFMoJul$Y}@85R4 zM+U*4{|re^Y_Vewiq2j{e!UBx@Ig=^e?{raSnvvaWUv((G? zmMkfa=E=>;`E`NYm+%V^k<{9*xp40vfB850{VbN%N~-hOOdnE`B_!;_+877x>C(SF z7mWCCNdvfs+J94B_#eMS>&B0T^RtQoRSR@40_|P??aFz1c?Lp3N_i+{!@KsP1oba0 zEF9^o;K5!{P0&;Yg(56a=rWPm<~E^&1ojejERu*S=qJqA1c&|4xX}y#lj=XT)_vPN zoAmhcPLM=35BQM9sA|DR=HME<7cc{HqyJz(ZE|c_TOeyN^Sg@13n#lmMn*^d^7Gw- zziHaJ^w`}!Cjegk^YinOYob7`88~iWP6hu_{O6y6dZ-;4k=R?&*$Q(rljC#K!<0pv zGp)jjLqsYu1%75kqS`W<7;0TF>6&Ff+?McAJ^*FzLFW(t>M4&bal=gWY#CxYPy7JB z_2-;GwXYXEbD4sNr467MBGy(p3gSp}F@STj`p>;dB(eSS@$?|12*g}QB%fAB3`*C% zwmrVuIy#(kpe2|SI$kI1Q^A6u6V&V0)0Uq*_oagwSx%t-!=)dW)nNC>U0b=Ga5Tku z)B(czp=nvUxmtPbQto+#UID!DaCos&-5%pt8;RTkYv_+qA+~e-^m{J>Dy>5$-84Cm z+&*lh4R{YIEpO9RehUy##)mBdDNpCSAD31kd8mTZCn9ozWF$Ze){IK;)d0|xTcqYE zeK2mHe+nnn??QBSp_xo0qRtGc1?W#HCK!!f6MV<-T>$%o0K+-kEj51vbsRkv&~(T& zY8E{P>d?(z=WF$ag3`3X7v#}*oA?2^Gdm>ipAxMm9Fga3P+XHeCw0Zy$R56W)LP0`lH&3tA%(YLz zBt;^@KWrLCWD^QH`Vw?VPDe$Muq-JzSQ(<{-u?z6ULAw3o}ihY2JlYho=%Q!HNX>x z9M#@WFD**9{%~ZpTlMj&r%Q*07aqR7kh@3nIxc61qMpG4>8XTGVB0ns0D(I3;k<~z zz;(-5g#OGs+f#MlxjWQ>*!ucM8p>D*by#3R5}zxj<{yC~u6m%7aCSgm4?^SGI(LSupbVDR10P(x$m=&p(obI2T~KYj$Z>P4xjT5^-MLimFz z#^k!>m$%{y1MnEd(23dns7rypm)$p9b(&&#>lfo5lQOJd-PUg6PTsDI(9{*Us7`2bmxV8pS=JE#TAB%_)Xu2Qx{$ecN=P>R1t8n2uTC z(?~glx&=3O9QT)M7Xqv$GJn?GoX5k%Lt0kW*l{*8lB2c2O$%636)HqMkaopgF=d2~ z)T}j=?)UHChb$62X@~n?YC({TcE9uIfuQ?y0KX%jKdS-ojlU8fj?u}rr}9)fywlgE zuZ|0{nxyS6b*1aWe;NpBq;ZBTg`(z`!yw=y(PIr-R+Rfl#4CQZmzJ4vdtslEO3*vd z6zuHmG_RXC?3t-1w-f>iFn1R3GBoiPZ=9BRM;!44PFD0P2pdKxYQ%A?)Tl zhjzDUC>^0v9=Zq{jrc8su^dHSRIO0}r+rU;|4uXcjC7QwK5mmqQOd|x_=$l~Z^iC* zm6Im}iP(p}XHaGjs%{%bPc`QFerW0qSV499E)Jt3xCS>rzXDwypTBzbYEC&42@J5s zuqUX1?n6ihyZ!*{VSKn$$6fI7>O6Z6i)Lx-j?bpgME}_sQE`Suu+&tWxTGgf zp4{c9$7OoN*4Sf|4Eh-o`i^=-3+U-8 zC@7Fw0CK??>>H8dmaw02t0(fJ%!h6ehpJ99+xnr?B*+iI2_-i^&9oiy4lD8GPlW%B zyEGIeY)mnP3KCR8+!`s7M1}wl@2lQR#2PN}Fy|~teP9lPn__v=($aEMg*(*&esl`m zThHVpNpbV?mOmIo7^4K2o106<07UT{>jV$%wtVyAW-EiExTGW=WgFx;&0sf>jPb;B zPEQOsBAQe3#&+%jHZx6B218820Tz!MKy6aaV-XZcC*>%GNdZOLnj{DorC{w44KxbE zY3jpR)i&7ruuJhH^^uN)Lc0-BLDX%^maH(yjCBZH;V1d_IgcGTFo=Z7B575bQDqn( z-k_SOEmkp}q*s^-kQ$YH1NgT0%39qjgzB;O`8oBg%|dQs^qTUxKR)Jphi0yP|tbtL%@fQjtFFOv+_Ve zYrt*f9uB!|>kka@dBy#hJ(c6feSwT>H#U})$rGc5VgvuJnqnZwr&pj&S^_K>z%!3P znh>MgWk>jgxK9WPa5yOefsMm;B=Ibod|Pkj9yi}>DfIAYYHA|Ro){t32Zw(C;m04% zKzr|X`TVHOW8O_};$kH^jF9xG%E2|{1;w2pLJ}qO7rRTW4D`$3fU8%o#6ral-fa~I zoY&da71gNgQX#RV<$?4PAY(9p&(6F7mxgtA7(;#7L63sBoZq%yfRffX+wkB z)2B~?+ICtgGf4acIIshO)jHb?Njfh-e;jOHRE`8U_mM@+Y;`CG_%>`POG?A;S^U#a zRWp;L$~qH3{8(Sv(lH}gNvUMSea=2hR@j*lck+$*-GESiOC_@bg*sq4V zP$fw>zue;|8o;oh;+N*g-rNWgA@2D7194?MDlzv^c?Bk|!1l|4ab_P~kSrA#rE+0X za~tpsuzzeJZxB2sE4M4Cz)V1#;bBQHNZ>5iIp)cs+4VW}INh~=9-a9aiGg@uLnHjKoO#saz5 zqA5ih+d%pF@#De5b*0!-(fBDT9|XUlso8#f`*hW;Wm{e>;EUwJ)NBT?!b$<{tj8Rs zjg7mdwXEG>6^x?V6XRkI!MZohdKm^|Q@4g;)iL||0aIr-3DyjR(CasE-r*Q*%~d~t z{yaCzN=!^e5+3;EWd#8N0oafN+t+d_{ki1@g>*0!HK15IK;s(*fxrY63nbTfS_cngxXcy588pycXs>iez}~rcNmwH zQs9{}fr?3nB$_%fw=wRsQ!H&spE^p+$@Z6O>?7?0?YLsvKNuIb315W5AcXy$-;DFG z3X8%OlvKlpv6# zowT2uadMkzbs+8Esf&M7p`a8C$VhV87f7m&(31^OJ7KuI8jBJI-LNiv;eZ+ge`=2U ziXe2hr*{k-UeH7%CfZyoALJtxBKGFNcyxD*_An+Gh}x;g?g`=XR~zRzd!f@JO4R^e zbGEw7RC&yevLm}319M7hBBfpuaZt4H_Nj&2fb^cO3DjlHS?422O#r*ggSkcpsgC#X zW_XPCoFGdAtYYPNX8j^j(wk$)g1^5^j1Po~hfcwJFjE)q8vE~8XAtP73J_dMJ>niJ zJr>l&M^FHeqm3M7Htp>m^E#vlKsAH+NQ6Q+co@cl%Mpv$tTiXyeRf=(Oi-kqP7fDa zeflZb`<%VK+STQ(tJbfN14lrDf9OCJN0olUsx= z5XH@~!GhY(fAvnrg;KyZp$u~p`9&cwQVq;HV2MwkJqt`AHD#@`Pd zl?hunc&8~yUFE~rmiOMiJ%y4;0YX|0f&}ysGf~Soug^@U8dWHfQV}WVceY8Yw@oJ; zxFZRbWDLYRZmo<*3&mgy8;x3ten{;P25V7e4dSMtGf`6J*DK&}fpm_fIwO#c-m z#7s1wTC`{pbiA?q9e2SxzMm~hY(KUOKH?j!;=2yIDYx9bJ<@E1(Bic#v4r+R7~i&z z6GNvAq>;2m96!E28}iAu;m@x%k4{;^mR#-Z+QI4ir^UVei}3P~14mx{i1^}iW@4n2 zTg|0U2tQpIq3hzB|ykBg(<8MZS&RV&s{q1*6}3oY$EZJ zOhPyRuhdunNKHbw_aC$g;1&PcGT0)27U2uzA+cpBNl?s7J?yDXH&=)0*h?LIwT5*9 zWbk?k$H$yq6!vvRfdOK&O@c+{%WzTMRG(;SqW*s2L@ap(S<$GFI`n} z282)m^n{xm#U3E5^kdrx!xPw$2+zd3kS8M3 zAe^PQ%6K0qboKibU~^w>v=1;Pz!0}##qy}lFzBK>XoycL2)2}hX~oIWF;EX?CHd50 z&_C&TmScfC_aiI#WoBl=2;K=z9KxS0n|jW~RZmD^6nCj=T)dY55pt#`EL;?xDhubfAA;Kqni z0#!x|MaVS7;1e)x)T8zY9STILN;wo*FyD6iil=AXp=-bPwHFq0#5xSOjePo4?YsFP zn@35?3(P(!uFVw7;ZH~p{NR2ZoZPP~SEAwMRP_)e)zG-X_2FkO2~A6ZdMcJYOou*!^CX5On_d%|)8|SudQO)h3jj2kdN(^)@m!RN@cK7Lo49iwz z*Fage-5Q{)#-1)>e}TCGK%gE6Eb;N@8?5LjSTI$9iu&99c#0k{&fiEyFBzJ0ZW^Lt3k&mVcoLq-bJbLnS_7ehNL zat%FihDv{X(M}au4uI?WQ7Nf|?l;ueA3fP|L%n%YLG8vVhr(DHOZm>4iAP7m zk}H0~4k2v@mYWB#)P&nk7fcW3=D-09Y~ybfPQ|Kr)^Ru-tC7Ija%REog6*!D*$W`; zY3`v-)ec754g{mfG?_g%>^gf~Tl+4ccLZS?@cP_H#rijR;%cAK-M{hcy+>R+)J5POHQEYJ_9Jjt&Na^03=a;HWxuDV zMrM zogKcVHI1xw55emuIS(4XoUL)3@x$A5G>+g z}rzL=qi-itA)*$FTetpgePL1S6gu21q`iXVpMiH~R9n^lXW-vK9 zxepMKY*OeskY}_x_w$i!7yqH&Frd8`&2z-A!~zI1B1t(y>Z`?laW6g(vQq*pg&0HGXDE@gXnk#X9kix=Sx zAA8*)xg&jp>cd?j?(>pF+*6hm%IBj+sCcLJMR}+n2MntFNv1=;6*fA>Jgi$^~y&4xu?hBm5np|G^Cr2ckif$&GZy)3P;EU8e5 z1d1e@v`B1~##>uuRiI>4giA?@n|)7j+f*dPYKu6T2<%{nkU)g!`}gnbB4nBE`szyG zcd5{;m^v>XU8EzIlAf-L9iN&*x&spvTg%oTTH&BFh2YuMU91KT2Ws_kr0vu+s@DMb zcxrvtYNi-mgNPc52o#HaPZc}rIzGtVQ2RLkb|3p9#Nm%a7VSeqN;6@HzW`Em6qU0Y z=NB$#pcZ9ptAFNuRX8IU|$JiiZat}}Z(4;-jDV4JS-+t-VH^P&Y z_W-(;I#F6)0tGwdP{4%{SA|exihEw;ws0j!#!=7H>w+|%KHsMA_hg%&94rDpDyys& zpp`=&9n;9(B-WKuR<+k^V_-mqD!s?ChLQRdGFxX~UvTXQgCn>}KL#TuV)?>NryPn( zs@?~A$i8mj1Ae>n!nSH0IwaV$ar6%0Mzxww5Pi+W=Q|X z6zdU{Qc@Ha?p27RXn&^5#1N4mC>6<_OsE#Gs)bI9`{BC5woeoHqy(9EHF#zam&V4e z3-68OF52&aNJD}3Sy9nAm}`=1vD?DY9J{3H^~vSRfJ+c|h0DAa`gPx)J}cZ@y>qEt z4c0JQb0c6Hibx;b)eG12*-->yAW^T7f6=0=w$o@1kyP8i@EeJnM}KGqLmib}zH=$y zIDms9KIege)uj5hqu7T}d4DDx$7IO2uU{CIC;DY7RmFv}je~kN`J~BQ zFjn$p7f=C(O%X@jd|_7KCL21+On@3`(x7USiAiHik?>b#O=8KvS=AZz7^%P`=H$7^ zb+lqPd-l48x3UG4?p)e&Iux75Kl$m=ZD6iulD!VRy7qPZgJgMbLz>eCk7Q%94F)DM z0kTULZX#^8kL!&gQ2b=dfwkAbaNoiOCGH^8fg6hf{r1vtyAPRRYNHk98AN1*sxo%^ zec^QmGEtW-;raYL)EW=mQ3g);4cjlj+(V-}#m_-u)#DQrjUMyUdejWaD1Y}E z?a3#Q+*335b_TXuT>5Y}M7*|tGD-XPvU%sjFSlM0S=y;6< zb*6~;CfaW3M2x?%@l8PVfPJc8Z)-ox;v9_U?)e$e;H0DUffdglgjMDiU?CB_Xi0jM z^ecVdg{3dYbOPYl<@WHHUp0uXIokbqRxRv;ClN0ZJI2P2j^8Y6cOAU}fSMY)W8pU0 zRHN7hM4+jsTePTwrmjQ2-XuqT$5gVRnl$*^BpchJaoUg&GdW&RCD=`l7{X_wM5++P zmb-r&&#-EL@~)brZ~LD|Y4Fy?A>=q>WnsyXXV7v2Vk`q|E$_A9)7z9Pgoumqq>2$f z7ADc;+BX8KipCo~q~d=c^QXyJfsv?2Ma8Gp;t~(*LJZnYsaW1+`6lp#^w%*w5wx6) zL}@=|GYrhl;{kjgPPr~tyU37J~Pat!3w`fX0(hr4zFrgDl<$tW4b{q3K^g$CSlA zuL_=YxZff$B)uRG?@(O7c#*h5BgchTDc(4`Pqq=A#CTX00Mq4c5)1Bcy+U`K4xog= zvadJspPMM0@1^~W4rX#7A(O>|Q;h?iM~%@y|L%y-tK8=1rmet?s*-guOu~s8oUoEO zTF;|wHb>usI&!HpnoL(AaNzE>#srrhq}u=17HK-X08Rf_Z_;*ql4HQ#qxr~0f2&_9 zN72sPC<*;V4SD=ZZx8YC@*YJOW12}B8pNp*NA{9qfaj}WjQ}8IPSA%k0dbJ}P}2~t zm)5(3CPBq?6gNeFL+lIDQ%{&sq6Sa%qBj7~aR(YM3SQMzqhmmjQ@14JWmiLZ128cG z=3J2RVDN1~G$g~-U{k6rO$g*qG|km+bO66Xo0fbzc-c#cwkRlx7N~HePCl?K+E{&! zD1pg(O#QCd!Y@Z=X(m6~zEr6qqc2S);zB(7?5Vexh-|X|!Drs2s()vJkwJ;F1wyx1 z#^!^bfzyc^sWK3U3711Dj{{oifo{`)C09O+j=U{$3#9a5YNe!$2`l2SGFo~lF~NVe z>0o-x>(?$Tn}9gf)h=+BIyW-1JVFQO28uZDTeZR?CpNxbrDPc zP6y7JK|c#&B_wLqd}ll2Fy6g;N2#0WqpmD!zd?~5jj|*T4S&>M0OV~jckuVq>w<+0 z7hG>x*hlcj20{i99#o9|{Y25h0`aU4VH^^XV zMqv^!1bl-*mbZC`d4MW%Xw(EtskxbSZJIhzLpJlB8?33F=FFS1z@yus!OF9{e>r_! zqZ6*g7$7fVDS&Dg4peZawT>c&1SNRescNM*V5cYz2GyF_k+CrO5D^4cpR{r$doeGm zQdtKOC~^$!w1sNo^XJdau;F`_WTQ=cs4d@oZn?tUJ@-*?@@(4lD; z9y$qs=m`LI^tsqm0tJ(x3bF|E%t!6{Sd}LN_;qaIl9#JNyN(@>UmzhD_J4Tv)bU1P zZE+g60l#;-{EFd{IxwG78%sL6oV}Rr3qcJ4ZYz5I@9s$=T^b$*0TFINJI5@@4n`!^ ztgyP0HH)Ap7>s5NI@ZF=9jIfMfV}))%+X;6d|ILa*QNBIRUY8|srGfAsF%nId`(ku{11;S%qUuVB-fytvxe~9mx$G zKov8wYOuLQW+%XTK^`_VJjC*|BU2Plbocg7zMh`g2~wp2qK_RE-4C$c5>bN^9$h(s z?^3^e?!%shwMP1OS|E$N3ii$a$5q++(G~i<+yq-@I?@<z5BUQ4HsQ1v_qQzB7K`0-01#0*OicfnoZ78Vw?c%e;tNs<8%d!6ozsxZ_h4N!H0 z&OAcQ(WRbF;Ar6D`zo0MdG0Cb4d4E&{p`#Pn2d*r%DqVU;B$y4I2E?}SS7G9O$Ji)LVjvGA9~RYGuUo z=X=%)nvLXN1cVpwgX>9u?e3Z52>;<+=Zh7jcCovSoyeKpjv!7)KY(K9I&hv0%!uq`9@Vb@i{mJ^(L3X%2-i=Ylm$@pt2HekHL-z<(%^kJ)vJ@L-8l z%-8^dA~`)Ie`k}FljoZ_?ji|{FIfTFVV##(% zB97G{s#f@{GedVjoy~zFD{%UI022|Bx*0lhr2!9!q{j!n4^ z4PczoT_>#v+Y9xu)M*>x8lexT_ni6jC!oHgq{M?aPeXT&ZQ*trU&ZCZIV=c})W!t& z(@`)2YDn-YhT`x_`622OR|?ZrKdK&Ly*YZZI;x?xCV=ZH1E)lk6Lq1YH*mDVw_twG z1*X*=B&S+h69qr}MMQM0%=L6eiAY;-(rqNR8bw@}@>|sOV~|)w6osK7&|~pE7jv6R zD;CYAOoN^#kaEV=R>=AwY5h289TUNiGcFQ!^A~IcC_Oc-S%+r_1_nsIrSqBK^rTgQ z`A@OkKo7`v19jEbXs;Js*aU=1}V?s!d(~cu*8JZxTRdrXQ0sfL{uXowvS_S3tFXLe7w0rfFIX<-@R!O zIUa2()35C>HL0w&*?&{^noa?oBXlW@uSNu^{XmzvpxR-i(sU029{#l>c0myl`f-Dh zjo$NG(@;3Ne&2f5SIWa3dqwImvCxB4L1!U+{L8bk@alg}^jbjW|KRaj|I?TFsxMXw z1o8Yo;@k{fKOd#NaUjK2J86Y*itAAt;w<*R>i>WJ$g%(FG5I$eqyA0u;lCn^R@?)M zq$9AR%N{?*&Oq$e$#)iwmM^{i@#8s|4mA=J6VW@Zih7>7q8kuC$7Q0w91VtVUjxQ>&Js;251l^v~T{{3c>>tCuoLJus$&A zOrf#b9~JYEGxl-Uc#Sh`93!Oz-wP!;etJ)qDT=ztaedVE3C7Cy?UQe8=qwU>a$XbV zhn?L+W_|>=oQ~(Pae_0T{^;5P8vrLNB<3e8JyOnkEZl>RwX00#SIM**?9&*U2X-Jn zqC;LTryo?yen9jtbv!XCS0~fZ$Yzw~iCR@+$a2$4@`aE$0xi1a%K)Fg73hf!GYnfg zPy)RR=-Y!MG@k>Iqatjd7Q`UI3AtlHaUzJ-kYXd7j6pv;qkofO}1BEaM zuhGy``$2Qiv0ey0+i)_FlZ%U1`4PQrYiSe&=&do(SUxn5f zgHi_B!^m(8=9P|*0wJ8yGYuJ={OM?vA0haR(0UtpZ$Kz0UHYDj;Sx$cl&|5hE3x<5)YA&M$0i7H&c+zoue-+xIVDgqW z2CRgxO2CNxxd1r~VxD_JP^|eK8F7RNbwbix?Oy$tT8%(?;LtT~AhhyZ%?Py@JVG`y z=P*-ca^uDgXocykginz^fV!&l{d-2&6r6uhK>T63d4z?x} z2J=tOzkVKW7r;`bc|jVPf{AeajZLK-<5jqn09vY6H6Fr&cqf5qJhqJQi^1%YtUr>6`^j`4l?>x$JXB@!(lR2y`R>Oato`z;IU=ox^G0uCj~Q6XCvtA3 zvX^*5kROalh~lJ>uW5`}gMRESj?AdQNRmUF@jZB)BKHnuo$r7+7PGYRmMZ3r?3Y-5 z$FQN)4j=vlT;-|dxr!n9O^C-UOe;k{5a%{p*dv^B3kM7m-GuNUTKO5P6slRhbBZ*P zYV%5Bk`cMko)JdD%2fFW&;G>6A4^YX(5lY!yk;lNYTD64SuiQKKxd59L8y!L`WIIc?AvHutufmVuBJUUa?0KktSCb#dx!Pk*ex?)kw+m)Ea% z?V2d?oG!r;AVUXB+^V*RyFRFtoIXm1U}AMJQREbC4Y9z24`Cnmn|b*&HZB1;j#6~@ zQ^|(Mt0-ee+aeK-KTRkKgD)V63qNx`RCEzEm)_;4F4`~ZFFxJFsTv$`UXtGphZ7nr zL0QotlO#C0MC@#C4vx+VwnC<+g1Y#@&~Dj$R11+6`jzvPmoGbJl@5u2dxl2Sj`-dvR( z(oSc8KV@K3aHB+6lh9k}?gp<%bha*N0}^+M)keoK3Fc~v+SEN^SXyxT%9UOusd^ew zIzI#NrvnxuU4#p$H<}7*?0JjVkN2~?e*C?7iVpBV0vAEQJ?mcE!zf+yoifCOP}-p2 zOgAiBgV-Jqy={#{h`XkC*17#uIjJY!rJW7#l!`+t#&QA667PaW(II718d7MTU;SV1t16r;@=6GNRQk%WmMhpRdk?TI*a^CF#Mr>l$J z3hK$U${)WIr4Sm2kuX8Au_nB15xeU>@NHxppzI5p>mjz0*5#+sPtonq-dutZ1$b2p za=K<<^sQ+J!k{Yrm$j_Bft_#=77JJHz?`fEr_0A`1?GlP*nlFx=#$EEVSAB*LFbm* zjUrZU5~Iu9fIF;nHpZ|0S}qMPS$50oV1q}S?$#^?QvV#+a2Q_#|M1~;zAIb3vUr3G zARIc(O^`YJT61@AUT{GAZdM!=&4DGDOD zwwIYWk@&D&j=7C<(j9=s#rg_f1mQkXmm{^xB%fisg+>hF4o%4PC)Bm5j zZvPQ;Z0d2Giwd4Gq}Gtuk)sTH%ybY_)zCTc0g4EsfhoBAHEu_@DaR!#N>2XyrW+Ip zLB_>vcz4k0HnGZdJdV6W3-itHuC8rzbV^XU8}(x2&2;3hqTOewzDt(aUlr_W=wAJ+ z<%S#Ey>%R?RM$u_G4vp*xC)6cNJ1=Cpu}JxA`*(E2r?F4xWEMCe#4vX+qOM_PrMR} zJ4CpKH_=d4jvPK*1si)5b}^Ya%y1G7a3H8%_KgkznT$wvU8mwVV0OfDKvX_#iy_Ng z!tKJQ?(xW>FjB^$Xe2{Av^mLQ>#y-hIDCxoZ$TuPM{$tSW0TRbvGQ}vkRzSmw9+tL z5P|5bOxPHwuc0kxA3!bXLfC$wcXv;K(SPhnL>3^m3TjDCIXVlFK7_PbWY9YZ>y!b2 zo5=)w)MlFbCY`x`n1TB@91@0uDSx~{3#Pt`kM5w*OkswjrYck=i|aXCB8+i}~)M3S9wv;x!@LU16<=>#KKLgHXC&>EYC zwV|`KGZLJomjKnV;CuxO3yIoFGo^|Ut&Q9`g3V)oHq+@YIw6VGC5aa|%L)6UTn!Xg zYHK2$1SA=}ybL;+>qHTNp6dMrKLCq`9CVh!V9Nss-^Sg*6!!qQHZTD52$vcUtgOI9 z@b#q?&!I6;7epbPtfHBKRDv$l5^Q83aW+bWHdsMg2Z(J_c7x1E{P z*bmnOIXp-`!_yFR4`R?4;<}JZ8L@$6f0}wFaSD2992%u?TxA3ne>^%`=omjN5jx4( zAErZOSDc{8s6#s+^>bjN>QKOpqtF1GC|1G%*}q?4a#?0W6vypT2}FZa4GgJiJs-AGE_xNWNh@8VpU@ba)?tr8|zLCWK5r z0$Lo%bwQ1!x`6VJ!SkA;FLf`NM~EDZ`?I8&0PW*}1c*n4uQ~wsW9e`>Z;8mDBjaEw zj--17&}LaLC5m1=H5i487X(hf?)UE>&;pO}k*ooJ#L&iI52-=NJ!Ug<7XBBUq;mpi zrIlM?F34sKCpJpPEnn@ba=>&-5K=L$IK@_IH7|P?YG5_gEmX>*q}s7-mm2O(o#Y=l z=<*?#pet*aqGT9EMU2rMmA@lC>bd;7aGIyD(|d>)F^+Z=QlGM$tR zC(t;a)W~E5GYRY{M8*&IQ^tLvi8E}Jf%52#FDyfO?CU%n!$n6qJZ$!Ws17HZUor3F z)9c83O2j*O3TpYMokUCwP#hf|4h0!6rjv|uJXVT9Awb^QITSw67zXF3Y^+-_3dH}@ zLDuBdrL$fT9Vr-|@xJP`&Bzvhm(AF;D09gn4lsEf&OnrX_o-196U&ICiH3sgYEt8a zPC1Ac#iBWt`zG!p5ts{#!xJZG^TK~9^ha%&t!-W$gGs`ve6u=3?1EG|J{_l#ia~zb z(dtfQ)Tg=gMz#lWAOT^^e!wdq333!%C;|1%Lhu$nc(C?*Q<0nNFdP+l%8wqr7i4_b zwZ{g9v!6or#^1PA&PQ26W<&6=j~guiFsfqSrm!l8Zo)(~o1B7wo)*{WclXag3ND zIwOt_n8faW*dq+52_aL&!Zm~FzV;pSnA{Wuwngl4)OM#g`dp zqn!j0j+I@AGKfJ_WGEKh`5@*T2Gu@vv^;bz7tg7XquU5sP-`z( zHJp!1N68uX+#SY-HUa5IP7wGM=wL={5vd`5$k6Dm#W|eV#v1Tel0Z+V2zp79Y)T5` zUk(LG`nxL+Uo5hnXi6UTia5V>^=MD?271fg`$qvvM~>jeb2d2Qth*u7;%D| z73YlKM?WMTVSsg-(dSF^#`Gf4_`Q*eX%K2fblwP{ClHg^Wh=%^Uy{;gj^a;DhZ+!> zPW}+T$8J*NJaroc0G#m3vk8u6(aGAl2+Q4^yLMr{$^n00S)I%`p2E?~^oQ-xzxIVK z_{IMtC06mS=p|2TOZRO$`QNLo-hDdo_|!U24lei)uksK`_w{24@3)q5yz`6g>33jH z85}s7X=5Teik{TIg*FUBdRecK5KgCCf|jfRj!<;P2Z;xMvh{*d6d4D3=*&)6oxz0TQJsG|_fAoP`>JX~sWFGe52;kus8 z?1?zyfshA|A+DwdsOjO!m*=l|<4F4m0wUDqhx3r=YaLF8sg~gE$Z0wj0D$5Jyk+1i zUJU8Dy#Fa@r&iXv`^3}T*eIrikf#RYOC7$rLMQ9o=4kmMWizA}!a7JsQ7>K`Zq0X5 zCkqgypzKO5tNe#B9BGi104bX=MY?J2^E?Tp5bEeaXh;L;MLjAchzp)Mq%iPn*a6_X zDmrl(B(+wW&DnP+0FtGF7}Wp}NYi7La6FC!TVj!0jE96zKl`3iDFHn?N{-Su=8ia~ zY()$ZX>U|#f=3}28|Kb`W$xwZCos1>f^b32;lQj7^q=@H4-@Jnkld*L;tzb^4{@0Y zBJy@lY8z$IWv?vRb(k+-e_3?@>NgyCk4t3!VUF-~f;QuZQjq|hEv{ZBVP>r31pY|ri*-Z|lXCvN z@Ev+@RsJNYrgm61rO$N?3KVUX)Jde-%?Vz~H)tzaXENf7RVJy3g>Xosfe|aM$Wz?3 zFSlrSIr&Ia~szPV*0t`e-43!`c=7s6F%-+1@EiN^atNRrqFV$`$) zD!L-q&A1B}pJGNw`3l5(4g1nGhk=RL;7dsJ{)AO>=lb>0v3IwRu(YxY&&tZm**9vu zrBG#}-I|Y7kE@xsV$`w4l7ElmZ@N@0f=L`(Y^oO^tf>6WDwco8}hqQ-qu$(2@K zHnkA9Mp^AbK z)UlTy0Q+%b-G!Bw4>zdytm4}V0-|Btbz#24{d&l{Fea~rOh4GSqY*B?x~|S6(J8BH zkGQnG8S~Oi>Hl#w_Tn5az=L#&Vv-(CFXThoT0vo=dCYqAf1$Btd`l3V|1b@WBeK{2 z+U4&sMR}jzj3aQnBU4jTa|nJ?z?Z0rayN{lu@(Z$s6!p)gj|FhCfFR#BL%&eI!adq z;$r^RTnx1!Xk!jKSr^Z$j0%}VBN5p05wqYn7t)ivvCZc|$SUBVG+1Yb96KJ(*_?an zxP+1kGjfBhW6Q+@kptMFAf{bCD9k9g5F8_vjc6nZEtBMNcjFG3(-SMO96sOs(r2Sq z{f!#{1EH9*)g<7UNDKuS%u_$hO~IB{uVe(t`kyFzH6V;8%h3c!le-CUDwN7K#FyZa zzx7vA(f|Q7=dD9QB6HKB0{>byaz@if%9mvy5OoOIrDL+Is;b7as?h@~hdhM@%%L;u zy*h-t=2Y|w9_D53moT0dvWa9@R8&wg8FA)$te}&!Y*sp^#m10>eP1D$OA%uZLKzBA z_qgVFJvrX!OJd$$ShC?o z^-I|lNK{IySHC`qhyN7N;g{bAh_#r#P8g>y+Qa;YbtzGK?nN9~GIwLurYOG61b*l{ zj7?9$v{Y3iUyN`0pEkzP0ihPH_=u)g^3nt`!)klXd1c3ssC?nGN;!K72+Q-V`~~~G zAL4KEfmNoy*QYF)u#Qd7+wpBsImS~6rQr93KAm9rO6c(F^J_Zm6$K*jNqs;V+0 zOOjg<8&+%kbWq%}vuytD)%m+XvgFKT0mMHfIWByk5`G_!L18RvV9f*{dX)#b2w4I@ zoUNP&p9#Jx1(fLJ?9=|zrz~g@d56|N{L5<1!6^C4(9kZ9uW2#m9aXwSVnFlPn?iev zO+z+27F0NKxst`eRUGa77ZS@>9l@T5^;M2VV5)hTP-SW)%~T?r&sqREl}aoE3;7sf z4Os9EZ0s`B2C2Dg^ZNgoZsxJ62XcFG`2T4bu2$du#y4-v>ddmhNOdA5oD2ei8t32E@N#u`b`A8v7F#+?f9CBHlwn8~d zGlGhmBH$Hu-J!?{^wNUV$!J2P7pz<>G(~aSlo0fzED|C9v?n}gw;$Hk)g7!Ud5Yjm z6qXv(HzrD+0|mBNLT%C0ip`kAJprND5u#IXrPAX>Gd2QA+7doiXmCHedcUk_0lRpZ zRJ>$>3o(9TzZTFQaIFl5`B@4~K4LyFve2W{8TpM;=l+{s%59+&8`|Ovs~5_F(@@s0 zkA=j=Y>EfS94hBNzZT151$4X2%JdB@_?x#Jmt-=pYRe_0Yd(Y=E!Sp-VTYR;3IDfJj=G(wy)=n@KAzLOq+?^rGpbDK54U{pR# zRW+7MxsmV?v7LyKP{dl2!37a4wkoDaekr&c+xmF&Bvyk59ez9SqdKhN#bD$Yd~3~V zU@Rq+h`uEPXGm{+8GsX^v0hi38Ekol2|jV+lI9(|5ZWR{v!g!nif@N0RA#5!!YC*ot=ZuCz&tdD= ztwMrS{xFNXYfdG?WCI1epSsXIOuciR4XLp}<~(8a<) z0teb!dA`)b73dy#$66kUAf2{n;Tikc(Nom?>8FCgO?}(y=JUU!QT~~8F?w&^{W5uu zkM_1OV4((xo(A(5@bKP|sS#w3L_F^XkiM*a(|aCeS8#@43>6r029%{O8=%r6#JRt6oHkk?dmtw;imOG#<2 zaI-KVmCvit5jZlM3T<4fAkgdWArNVwFmZC*3*q0Qxw#F^J+!zhrFed1N*L|B=6~n_ z7=6oA&GgtuTjEbsx8jgGF8P3G4ll&Z$A~ zJf^La4J{5hqwX^8P67zy>GZI2eyix+RjzKFB_d6Wlh82Dt7Z!YLb zKx!JhBYob%TZbr_>eQXF;mWXO>riecwDR?uxkbXG@$t{U-X(V+X|q_!tcfQjlGD?X>%hw(|O5>)WdZ@ef;pH;+%h=NuJS|^4j|P zLsjR-;<}+svw>Q((VeR|(KNnf&Ag;$zDihxWjo)wbMVA@YriR};n+P@-}ZCU%s)(& z5MGK0oxS%%=1fvx#s&6tOT*rrZj6{Y>N1ihT(Q<0*`_^!1zj`Bc~bwr<)AeN_asS?&+#8XQ*Ur_(Q8 zdh;~#+zNdpJ=?8~On2~aH*fa8>6#oH-DVGE{DFqXz{tD1m|XJH%h6g|eU)~+$3Fvi z_07-Ecl^@d$46vErsbox-d6ssPKnet?4HMDZy%q9ix=-IDX~p*3JADVzU-uN)DSo> zG=sYxlRDl_7{2#@Ny*NjASFd;C{t1Tv)I%$HE)8@B)lSCVWN@IUv_72Z))nnT{_L7!-1(zR$&++(BWF=y>UXr@*hY9uBlWjAef!<|+< z?!jt9*DYH#88P;)an!GmAMeg{pFL;JZtH><<1-!i18CuVbl$LG1nBG1<;!Pp-I^Ke zQW}Vcdsd8%i?cLp8y?f7|s+{9< z&CCLK0v(>Qjdwb>f4QUMkfZrQQFzx1-n;d5-DqQD4de=rhB-x%!Pgh%W@xj`e`hA#Kt!-d1+$v^xOiav~xVY^WI<2}{ zBL?(86cx40CeCc-$}!;w4!|sJZ);m$(0#8!>p1C)NB zlbcDVN&k>reUq0ePM*{{6%%tJElqXyi4Xc5w@YxDa?~Yx>Gy{Z?|t#Yv%RB3d&-ou z*j2A-bOzYj*?kgbvB1twDb6-HKK^sW5*fzX_6`nUVeQXtS~~^|95~zFex$pBdW6-O zowHA@%9_@)MdyH-*7n&aTH=NbD|~Fx(%$~^@L*WR_AJi-Pd@!rY0#iSn z#qD)pI_?WBqIyc@4DCztzU{(Q%*)}h7)nLfjU?{`Y~XZiIvX6I%*|KpzyJB$th literal 0 HcmV?d00001 From 63133adb5ef81557086ce55853236b01573eb434 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Wed, 11 Oct 2023 17:51:06 +0300 Subject: [PATCH 44/52] Added CPU only version in the docker , updated README accordingly --- Dockerfile | 22 +++++++++++++++------- README.md | 36 ++++++++++++++++++++++++++++-------- main.py | 14 +++++++++++++- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf77d8a..343a0f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,10 @@ -# Use an official TensorRT base image -FROM nvcr.io/nvidia/tensorrt:23.08-py3 +# Argument for base image. Default is a neutral Python image. +ARG BASE_IMAGE=python:3.8-slim + +# Use the base image specified by the BASE_IMAGE argument +FROM $BASE_IMAGE + +# The rest of the Dockerfile remains the same... # Install system packages RUN apt-get update && apt-get install -y \ @@ -8,17 +13,20 @@ RUN apt-get update && apt-get install -y \ libjpeg-dev \ libpng-dev -# Copy the requirements.txt file into the container -COPY requirements.txt /workspace/requirements.txt +# Argument to determine environment: cpu or gpu (default is cpu) +ARG ENVIRONMENT=cpu + +# Copy the requirements file based on the environment into the container +COPY requirements_${ENVIRONMENT}.txt /workspace/requirements.txt # Install Python packages RUN pip3 install --no-cache-dir -r /workspace/requirements.txt -# Install torch-tensorrt from the special location -RUN pip3 install torch-tensorrt -f https://github.com/NVIDIA/Torch-TensorRT/releases +# Only install torch-tensorrt for GPU environment +RUN if [ "$ENVIRONMENT" = "gpu" ] ; then pip3 install torch-tensorrt -f https://github.com/NVIDIA/Torch-TensorRT/releases ; fi # Set the working directory WORKDIR /workspace # Copy local project files to /workspace in the image -COPY . /workspace \ No newline at end of file +COPY . /workspace diff --git a/README.md b/README.md index d526407..683a7de 100644 --- a/README.md +++ b/README.md @@ -34,14 +34,34 @@ This project demonstrates how to perform inference with a PyTorch model and opti - [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#install-guide) (for running the Docker container with GPU support) - ![New](https://img.shields.io/badge/-New-842E5B)[OpenVINO Toolkit](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) (for running OpenVINO model) -### Steps to Run - -```sh -# 1. Build the Docker Image -docker build -t awesome-tensorrt - -# 2. Run the Docker Container -docker run --gpus all --rm -it awesome-tensorrt +## Steps to Run +### Building the Docker Image + +Depending on target environment (CPU or GPU), choose a different base image. + +1. **CPU Deployment**: + For systems without a GPU or CUDA support, simply use the default base image. + ```bash + docker build -t my_image_cpu . + ``` + +2. **GPU Deployment**: + If your system has GPU support and you have NVIDIA Docker runtime installed, you can use the TensorRT base image to leverage GPU acceleration. + ```bash + docker build --build-arg ENVIRONMENT=gpu --build-arg BASE_IMAGE=nvcr.io/nvidia/tensorrt:23.08-py3 -t my_image_gpu . + ``` + + +### Running the Docker Container +1. **CPU Version**: + ```bash + docker run -it --rm -v my_image_cpu + ``` + +2. **GPU Version**: + ```bash + docker run --gpus all -it --rm -v my_image_gpu + ``` # 3. Run the Script inside the Container python main.py [--mode all] diff --git a/main.py b/main.py index 058d9f9..1ec21bb 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,14 @@ import logging import os.path -import torch_tensorrt +import torch + +CUDA_AVAILABLE = False +if torch.cuda.is_available(): + try: + import torch_tensorrt + CUDA_AVAILABLE = True + except ImportError: + print("torch-tensorrt is not installed. Running on CPU mode only.") from benchmark.benchmark_models import benchmark_onnx_model, benchmark_ov_model from benchmark.benchmark_utils import run_all_benchmarks, plot_benchmark_results @@ -79,6 +87,10 @@ def main(): precision = config["precision"] is_trt = config["is_trt"] + # check if CUDA is available + if device.lower() == "cuda" and not CUDA_AVAILABLE: + continue + model = init_cuda_model(model_loader, device, precision) # If the configuration is not for TensorRT, store the model under a PyTorch key From 64773df588240e94827b6ef405afdc29f35f7a66 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Wed, 11 Oct 2023 18:02:06 +0300 Subject: [PATCH 45/52] Updated Dockerfile to use only original requirements.txt --- Dockerfile | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 343a0f2..9050db0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,20 +4,15 @@ ARG BASE_IMAGE=python:3.8-slim # Use the base image specified by the BASE_IMAGE argument FROM $BASE_IMAGE -# The rest of the Dockerfile remains the same... - -# Install system packages -RUN apt-get update && apt-get install -y \ - python3-pip \ - git \ - libjpeg-dev \ - libpng-dev - # Argument to determine environment: cpu or gpu (default is cpu) ARG ENVIRONMENT=cpu +# Install required system packages conditionally +RUN apt-get update && apt-get install -y python3-pip git && \ + if [ "$ENVIRONMENT" = "gpu" ] ; then apt-get install -y libjpeg-dev libpng-dev ; fi + # Copy the requirements file based on the environment into the container -COPY requirements_${ENVIRONMENT}.txt /workspace/requirements.txt +COPY requirements.txt /workspace/requirements.txt # Install Python packages RUN pip3 install --no-cache-dir -r /workspace/requirements.txt From f9d7dc92172e96295aa776896df74b96ed63fd94 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Wed, 11 Oct 2023 18:39:49 +0300 Subject: [PATCH 46/52] Updated Dockerfile to use only original requirements.txt --- benchmark/benchmark_utils.py | 1 - prediction/prediction_utils.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index 532a4b1..b49ca67 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -6,7 +6,6 @@ import seaborn as sns from typing import Dict, Any import torch -import onnxruntime as ort from benchmark.benchmark_models import PyTorchBenchmark, ONNXBenchmark, OVBenchmark diff --git a/prediction/prediction_utils.py b/prediction/prediction_utils.py index 25a5e7b..7b0f2e0 100644 --- a/prediction/prediction_utils.py +++ b/prediction/prediction_utils.py @@ -4,8 +4,6 @@ import torch import onnxruntime as ort import numpy as np -import torch_tensorrt - def make_prediction( From 2f81c9a5c78d9f4c7ac3d592a1024aaa12ecce62 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Wed, 11 Oct 2023 19:17:16 +0300 Subject: [PATCH 47/52] Fix cuda related parts of code --- benchmark/benchmark_models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/benchmark/benchmark_models.py b/benchmark/benchmark_models.py index 6a73677..c56064b 100644 --- a/benchmark/benchmark_models.py +++ b/benchmark/benchmark_models.py @@ -72,7 +72,9 @@ def run(self): with torch.no_grad(): for _ in range(self.nwarmup): features = self.model(input_data) - torch.cuda.synchronize() + + if self.device == "cuda": + torch.cuda.synchronize() # Start timing print("Start timing ...") @@ -81,7 +83,8 @@ def run(self): for i in range(1, self.nruns + 1): start_time = time.time() features = self.model(input_data) - torch.cuda.synchronize() + if self.device == "cuda": + torch.cuda.synchronize() end_time = time.time() timings.append(end_time - start_time) From b6ce20fd3cab29eab9b53bec00743bedf434b142 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Wed, 11 Oct 2023 19:41:18 +0300 Subject: [PATCH 48/52] Fix cuda related parts of code --- benchmark/benchmark_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmark/benchmark_utils.py b/benchmark/benchmark_utils.py index b49ca67..2957d2b 100644 --- a/benchmark/benchmark_utils.py +++ b/benchmark/benchmark_utils.py @@ -42,6 +42,9 @@ def run_all_benchmarks( ("cuda", torch.float16, True), ] for device, precision, is_trt in configs: + if not torch.cuda.is_available() and device == "cuda": + continue + model_to_use = models[f"PyTorch_{device}"].to(device) if not is_trt: From 84a3178947069f7ded899ed575e8b331a128bdb7 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Wed, 11 Oct 2023 21:22:02 +0300 Subject: [PATCH 49/52] Updated README.md --- Dockerfile | 9 ++--- README.md | 68 +++++++++++++++++++++++--------- inference/plot_linux_server.png | Bin 0 -> 25959 bytes 3 files changed, 52 insertions(+), 25 deletions(-) create mode 100755 inference/plot_linux_server.png diff --git a/Dockerfile b/Dockerfile index 9050db0..f0d120e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,9 @@ -# Argument for base image. Default is a neutral Python image. -ARG BASE_IMAGE=python:3.8-slim - -# Use the base image specified by the BASE_IMAGE argument -FROM $BASE_IMAGE - # Argument to determine environment: cpu or gpu (default is cpu) ARG ENVIRONMENT=cpu +# Conditionally set the base image based on the environment +FROM ${ENVIRONMENT=cpu?'python:3.8-slim':'nvcr.io/nvidia/tensorrt:23.08-py3'} + # Install required system packages conditionally RUN apt-get update && apt-get install -y python3-pip git && \ if [ "$ENVIRONMENT" = "gpu" ] ; then apt-get install -y libjpeg-dev libpng-dev ; fi diff --git a/README.md b/README.md index 683a7de..d9a1a1b 100644 --- a/README.md +++ b/README.md @@ -6,33 +6,42 @@ 2. [Requirements](#requirements) - [Steps to Run](#steps-to-run) - [Example Command](#example-command) -3. [RESULTS](#results) ![Static Badge](https://img.shields.io/badge/update-orange) +3. [GPU-CUDA Results](#gpu-cuda-results) ![Static Badge](https://img.shields.io/badge/update-orange) - [Results explanation](#results-explanation) - [Example Input](#example-input) - [Example prediction results](#example-prediction-results) + - [PC Setup](#pc-setup) 4. [Benchmark Implementation Details](#benchmark-implementation-details) ![New](https://img.shields.io/badge/-New-842E5B) - [PyTorch CPU & CUDA](#pytorch-cpu--cuda) - [TensorRT FP32 & FP16](#tensorrt-fp32--fp16) - [ONNX](#onnx) - [OpenVINO](#openvino) -5. [Benchmarking and Visualization](#benchmarking-and-visualization) ![New](https://img.shields.io/badge/-New-842E5B) +5. [Extra](#extra) ![New](https://img.shields.io/badge/-New-842E5B) + - [Remote Linux Server - CPU only - Inference](#remote-linux-server-cpu-only-inference) + - [Prediction results](#prediction-results) 6. [Author](#author) -7. [PC Setup](#pc-setup) -8. [References](#references) +7. [References](#references) ## Overview -This project demonstrates how to perform inference with a PyTorch model and optimize it using ONNX, OpenVINO, and NVIDIA TensorRT. The script loads a pre-trained ResNet-50 model from torch-vision, performs inference on a user-provided image, and prints the top-K predicted classes. Additionally, the script benchmarks the model's performance in the following configurations: PyTorch CPU, ONNX CPU, OpenVINO CPU, PyTorch CUDA, TensorRT-FP32, and TensorRT-FP16, providing insights into the speedup gained through optimization. +This project showcases inference with a PyTorch ResNet-50 model and its optimization using ONNX, OpenVINO, and NVIDIA TensorRT. The script infers a user-specified image and displays top-K predictions. Benchmarking covers configurations like PyTorch CPU, ONNX CPU, OpenVINO CPU, PyTorch CUDA, TensorRT-FP32, and TensorRT-FP16. + +The project is Dockerized for easy deployment: +1. **CPU-only Deployment** - Suitable for non-GPU systems (supports `PyTorch CPU`, `ONNX CPU` and `OpenVINO CPU` models only). +2. **GPU Deployment** - Optimized for NVIDIA GPUs (supports all models: `PyTorch CPU`, `ONNX CPU`, `OpenVINO CPU`, `PyTorch CUDA`, `TensorRT-FP32`, and `TensorRT-FP16`). + +For Docker instructions, refer to the [Steps to Run](#steps-to-run) section. + ## Requirements - This repo cloned - Docker - NVIDIA GPU (for CUDA and TensorRT benchmarks and optimizations) - Python 3.x -- [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#install-guide) (for running the Docker container with GPU support) -- ![New](https://img.shields.io/badge/-New-842E5B)[OpenVINO Toolkit](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) (for running OpenVINO model) +- NVIDIA drivers installed on the host machine. +- [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#install-guide) (for running the Docker container with GPU support). Pre-installed withing the GPU docker image. ## Steps to Run ### Building the Docker Image @@ -48,14 +57,13 @@ Depending on target environment (CPU or GPU), choose a different base image. 2. **GPU Deployment**: If your system has GPU support and you have NVIDIA Docker runtime installed, you can use the TensorRT base image to leverage GPU acceleration. ```bash - docker build --build-arg ENVIRONMENT=gpu --build-arg BASE_IMAGE=nvcr.io/nvidia/tensorrt:23.08-py3 -t my_image_gpu . + docker build --build-arg ENVIRONMENT=gpu -t my_image_gpu . ``` - ### Running the Docker Container 1. **CPU Version**: ```bash - docker run -it --rm -v my_image_cpu + docker run -it --rm my_image_cpu ``` 2. **GPU Version**: @@ -63,7 +71,8 @@ Depending on target environment (CPU or GPU), choose a different base image. docker run --gpus all -it --rm -v my_image_gpu ``` -# 3. Run the Script inside the Container +### Run the Script inside the Container +```sh python main.py [--mode all] ``` @@ -79,7 +88,7 @@ python main.py --topk 3 --mode=all --image_path="./inference/train.jpg" This command will run predictions on the chosen image (`./inference/train.jpg`), show the top 3 predictions, and run all available models. Note: plot created only for `--mode=all` and results plotted and saved to `./inference/plot.png` -## RESULTS +## GPU-CUDA Results ### Inference Benchmark Results @@ -105,6 +114,11 @@ Here is an example of the input image to run predictions and benchmarks on: #5: 2% lynx ``` +### PC Setup +- CPU: Intel(R) Core(TM) i7-10700K CPU @ 3.80GHz +- RAM: 32 GB +- GPU: GeForce RTX 3070 + ## Benchmark Implementation Details Here you can see the flow for each model and benchmark. @@ -145,16 +159,32 @@ OpenVINO is a toolkit from Intel that optimizes deep learning model inference fo 4. Perform inference on the provided image using the OpenVINO model. 5. Benchmark results, including average inference time, are logged for the OpenVINO model. -## Benchmarking and Visualization -The results of the benchmarks for all modes are saved and visualized in a bar chart, showcasing the average inference times across different backends. The visualization aids in comparing the performance gains achieved with different optimizations. +## Extra +### Remote Linux Server - CPU only - Inference + + +### Prediction results +`model.log` file content +``` +Running prediction for OV model +#1: 15% Egyptian cat +#2: 14% tiger cat +#3: 9% tabby +#4: 2% doormat +#5: 2% lynx + + +Running prediction for ONNX model +#1: 15% Egyptian cat +#2: 14% tiger cat +#3: 9% tabby +#4: 2% doormat +#5: 2% lynx +``` + ## Author [DimaBir](https://github.com/DimaBir) - -## PC Setup -- CPU: Intel(R) Core(TM) i7-10700K CPU @ 3.80GHz -- RAM: 32 GB -- GPU: GeForce RTX 3070 ## References - **PyTorch**: [Official Documentation](https://pytorch.org/docs/stable/index.html) diff --git a/inference/plot_linux_server.png b/inference/plot_linux_server.png new file mode 100755 index 0000000000000000000000000000000000000000..ef401ae21e9a3cf21b00da6f4d2fc12a2a1edec9 GIT binary patch literal 25959 zcmeIb2UOMfwl#_|#>4_K!A{W#Dp(K{P!Ld2aVxzD79dKKUZhuJBN7w^5v7PAy(&eJ zs-hqw(m}dN?~3$Id2{WYbMJj)yf@yhYi zgpG}DIpf%2RW`PTS!`_IZTj&C{K<}WM)(7^eYQumY}G6cZ5__p7_cdvv$Zm_v^6t6 zzro(X=AyBs#qOQrJ9ls2aKYBr>Y{{@komv7V5g;xk?jlW@FQ@W*k1K?if7S?BuFYH#h%v;>OzzegZ2l{&;Xx%p=hwKVPklROd{b3Ot{) zBeAhDJTgtA<_^1H%>q%4c+uDU%VZTas`uXw{H=?xf5KDuL*1P}uO2pWk>d6F=D61- zId|MBH1&={ZG%grxA0Ls0vnswRWbZ|`qrJA%@?t8^X8*sK?>`|E#42QRz~U--5MDk zeLp0U^7`eK#RjSMoWK6+*Br^_cy)#Cf(=Xg=F3B7Z0zk%w!OMm^HficThe}e$+)C# zk8-`4*vhTS2Lgn26`x9P*l<8eW_DC+Y_>N#G-~>a=?F)9Y;KhN1K;r3q2}zBYu5@W zDJgZ925AcAe7UR`D($8u8rkW7YwZ1l-CkZ^zS-H?(o-G3x_v7ZANlg-ZO0N`p5kte zk&@zK#-gRGs;iV_64OoV3>}<&f*bRFg7;k%#3#Bj$7hS?4t3S(deoGM$);v6WApjB zdjM;dFmO}6Mft`r{I+9#8j0H3CxwNDPd~Sbv>*7mQCvJ@d*g7dTJ#=^roFk7pZ|JL z+Z`S19yc@fEgp*zn`BuPbu(~}@#iNA&SQP-!#+GxaZjG?nwp&KNz7cf*b z*qC8DJ3VG&ZC#w^JX~~r`Fh*0pO55Dbp?6-`RCgqn>pdao9nmWJM;7N_rATeY5MbD zLY%8sMUC~>#bZnAwOclAnPO(z_WG{caXP|Bdiq$9xMl4-|81$+X`Jq}W14Ow#apgw zvvDjtxSB0RxxMhFM&$984Ar_MZGRaL_uSd>#%U}y=gO7-2FyhDSl<@63GK1IniE4q zL#LjFp3z7;d#X%k-Zjv?#kqfnc9tz8NX%UI>G?N&jG8zNwJSd?$38!{XwH)QQ+2qn zI#$g$qUrfXqtl@QJGGsA5dD5XhsCUPmcwvT z_d^#&wP?kf>9Kx|#M9~W^^4i|C*03k#>SyM*pSZ8U``A>Xw5N)^ZVT0Og{2Nz`dZLppnj!=+o&Y0%BrfZ`^e}X6U7tMTQq5# z9zTA(^5avzm0Vog#l^+x$Tep>)IGo0Jz)Obci*L5>{i@y=Gl8`ZIS)^_aD*4<5yyh zhclDc?l$bf3O(~Y%fOZ>D=Xi2pKSBYu6362jn=}Zo(}bxd;ZQW znm0as=eNZNFE`MgAIxYK+_Og=TktMOJo%x^&_3>cHm}2czI?Ir7t+fn~R%?}Xy!-re> z;`#Q2f!CL<4aeCjuBp*D^4F@GM+dLi4mOB-+T%9ox=hsi$jrrH9ZT@nFJHb46t~pz za>I3%$L6bq9lF+m@A(V=@ff4BhXNlmOzV<*W#*@RT;1F{+uFj69y)D)Q$5*XJa{5t zr-r|vM#s=l((T(nkkx22^DVPRnnDF%NPjOWje2T9lu#Q5@`c!xiq zo15!vdY<|rcwc+@kVW^)!DklDg1Wl8*E~J9NlVY&8XW3SoX?Bs4$gJ_8a_QSoP6mL z+w-ifcV%Tt%h&BG?a)w7u%-89hCGFy$7vWS^gs^c_A=fx2RZk?`_zWhW=6 z;>N}^8#iv8{&a;y#CekK)1?*KY>t=MdG>E*&@Q;ue+ZG(NHtWTOx@QSxiu=BB#GW4r(O;}746 zH-D{;w{9<7vV3{X?TxY>9UT!je%%tMlk0r?>G>mGDrqMZPsiH!Ryy}5X6Sq0;iD%& z<_QcB53f!))eMn#JI(7hdgs-vS7`<%t2|XYE_Qzq5D`(`aXL*|UHxv9QfN|uuKOvh zw@FjxQ$7JTVr_YwW7@B}N@uaCzm`)I0EHsLCc!{u+gdk-&;on_usR8vzE zm*EyZgh1W->41|-XJZj@V$VLeX~=WSZt<9RoBdkEJB?r5)A++k!wLm$k`uh{ zlVPWynZ*`(a%|JhbM=jAFY<{;gplx<%gM6qk4iBp=|fZqly=M7shc;jt20+ARGQ8J zU6Lz{IZOsVKJ9DH&WKVB>3hCpGBOr{PeEZu(awCB$b(ceDV zuNN~@eth(%I%0F_Ln-D%!}Ag2!zY}}>UpG`FHDd2(j)M=3PgaSL3-& z=dwDRc>J~>58!q4W{r5OYx|R(Vqz!F8`4y;u4ChkmM5~DxaV=cs*`o|nGHgh1Sbs*D(x{i@Qd9tGbX^4Dlg_b#--_Sy_RbhXbjLu(5l!!R;^|{UL^m zBI8`a56qN^2tKteTjQ!2l}&zk@9OKiPmLI;g&+Q@3U`v3(lH!{?P3?@-tSo6@%B!E z_Zs0>=cdL78*iqjrWTAy*!4wB<|Jun8zNpRl-%1fSdus07th)HEm3s6-_zlJqQylQ zncEzOnhcSK*v<2&y4JUMbm-%|wmos3nf!Xx_f)!xzuN=8_RpWs<86&}pP4s`Z7Oyh zt&kfxK&ls4R8(YdEgCGkKNE7~@Zr6#G7EoLwrihF$FAEG+!D6C zczAdU*XZGX80X~XmZ~X9`6uNsES?>zPb~_Tc5C3m_jW}9C~R(So}8WzM2xPN2^xKv zDlIL&Swcdhbas`HPLpj<`NlW27<*?LZ_hZGe|~*q+@PYOVskpLBy+6a2yd-$=+G-| zpY?1amt#xVZeRQDx8FMY`{Top{MCmX?eFR7=|0`NWu<`X;mXn#DY)F*1O<<;7SgFk zY)Z_ZpF4vf5dPRh*OrWok3ZwpQ&shKpZO{$YTmG?rk*Xs=b?RnZRxpF zKt=xd@5kU=>`QYQcv{3|ac}(fuS&;!`)cC5ccn`^O(lj-B|c8nOxCHP1>X|ts_M%X zHN81|c7J9K5^XhZ2Np_qx6udrvEibSy8>!Qv0zJn{8734Lx@J4xt z#c$nVx=8iX*#*SO}RSQr)waoc?Jy6s%3P6Rx%$+-ToZGIgRtK_>9P;QX4~tWa zzE$YMo4jD*4-s1VI3e1ZRtMeP-5r`Ha^C_D(JjT{>Fud_ET|gsrxaI{dq|Y+?(N5p zU(D+2t%}ClT<{p~?R8Qr+n6i*`q;d9={JkIxXtsIH%?7ASD($adb#m%z*d#tj_?+) z@!zQ}Q8Q>6Dt$KM*d5M;JCz>pp9cCG7@inzO%^eD7m=wrK0a$$IKj1QRhQN7IHORv zlYp8hQF!23Zf54$54Y^l)oryM>>nSN&2sWCZY{zytX#c1nxJTW;i!9lM#i_VU+?g( z+;MtOO=pIM$N0c0Lab_0@(k2lz{D_4XcS-Eaq971IbQj;L!!*!E)F-U#Ap1kfX zu%BslPSot9G7^%PxA*(X+3%i6qX^Uw17xX}3j?@ILNcVBM)_)1c!%$3GW1Q^(<>7ndl z)#f?HbNo`i3k&K~jWi;TtysBTOWD^Jv4qoHDR^&Kj?=fOeSrWj8lEc8H_Bdbs(O6f z1~IwM=Ax(R7C(LuP2`Rt?%`doE?@GbGPm&a7aOJ;4>diHt+i&3Pn_-_OzCqOA04g! zRcU`1KB94WbMW1}5rCRE*Y4h2V{XCW`@qh7?clKDU}dTst}dc#B9ciQfbV#I2~M5B z(XmL$`R|GVa}@&jR1DOoidwehER&g-s@2`U$GkzE083p`h?sc;-C?Wrm$p*LqD~QMfQnu| zKAH%LfcDNskG3^))hy@WsKYV5cV7GV-yf~oc{T=@sb#E7++lj-graY>^JdQ&T*Q*P zI!)kF`3fU3O6=l+srd%HEMV$r3=Tleoji~6(*(9RB}`_$A_z3c^(O(7Z5t2)Q! z0vw&4%=>k6`r`nYrB#+PlLr#U!$)IHwj z&Q|QGES22ufdm?-9Cq1_Pt%u><-Utud-oqV!U{~c`^Z!SYVGlPj5C5_@ku_9ZTa%$ z%Qdi5DE3489_y+yxh(3!k1Xwcdq)0IY@z>&eK@Ws)OY35!NI}RIZhX6kdH z+=n$r-gXW$fu;wf#=1X$4#x?l8pWg`Ef%oOqbwa9h~g11&Zn)pdvm+Z&YwT;B~XWp zE2tiGyC7c(MZxK8`^17xAr|vY*Db|m?0K_ivuA@HY;00=Z&_z}G_q~j;Y|lv%<9tt zWWVE}Mfxfpdr~66s-K|JY9fC=5xJ%>Ie)GaC)9SdM+H$vp|Uin8^syraKOSEkQ;+O z3!eue0>l8)3*4(iVN7*7AZfZ?|4E8nSm5fGTo+1%ZeLz5b_NvUZ=MNrYRa@;C+(Vf z++UDiMn(qbM+t01C8Cu^zPsB>LG@#cIe4n+Yyjp&I6A4unC%r%%7z^;I*jEnQvvRIeyazxWp?7ngoiQHrTPFJ2s2 z&MSQu6=fR3|(tX;ZgtB&7hg-VaPZ@h`R`8rtjS|pD& ztF}XQ(s4CP$|r*FAzzeKRjC!O;g)ef`|D=;a&QuR+-Dq_e>N`_%6Wte<{;2bITlkL zB`N>r&3!m0b#!|WU!tAWV$*WQYR{a0^7|k79(o@76#(~$UwabIKCf)MBY&zC#qPOt z=bXDk9BjdsP!3+YYI`|?;m~s>2M34J2fO9)JZiw>>1OpQwz__<@nFUPgxuHdwR(h# zHu~+`x3(dvKyd1JemgQckXjaZ{*4#tzG_68bO7(%i58dfIq(v3_%hp2`%wXu?~$K3 zJATG9=teUGNsTYovNg}7I_3_-d>9peNFX(o9FldU2r8m1sFdS%i@<(TWw&;28QY7U zk``mj*iHsCv>W^4iw9!ie_gZYf=2`;B?adA;LZH}{Ne|@qXo`pK5+t(Rf=+Euo0Wn zq#l2=wBtiaBSB~sd4Vf79lhUQm)zLcH0>DH$S;$$IbDvEs^HCg_Eh6V>xG1be*NW_ z^MIN9z&aL|7e_i6->TJ=*5WTV70>OxGn_x)a`oD^wV)%~2L{Xl_AH4ANHwmCQkMzB zZMzTlE)b9Y02nFAez4(z-xj5);VY$=B&BpHTPReV^j#~W|L3gR>DJbJJI=_XE|<@9 zo$`*4j~4*+zv1O2fhVD!7}D!dQk={}P4S`H45_O`Zjz>22fj~}BvbQv(GoxXhK zikewtnCxOTs<;e5po%5uO~(nSgkPm4`8qKnUM9^O-Hd`(benM1J4y)f3S4*&X0|Z<70QkVxwPj;a^ z75MNt?ZcHbyh3*eu#RFrQsejO0bM4B6rL(LPv=^-UHcOS@VZ_y!Qhf&p@x0 z+ty+A>DvpQOt>IDJ6gp#n~$>=W!3hI9=JN;RFzp%rm(ip>f^_b?E-fq3?N%*m|D*( zy2owWw(7|_+4k;kqqAqv%7e3%pBQQ`#qAh!oMF4oBc{jmC772*gs1#m_wQlC@!Ego z(upAEI0=l7)3Os!S#S1iVCQ1ID(X}VDvkEa;2uqJRzQ*(1CYI^{hUZfqR)YdRr&p4zB$ScYZ_d9W}SP_PVT3;sw zhF2_TM@=1qU`mG;9X&0MgR2F$pLr$^@_+!vc|AQlBp}M^fG#N+KY?(3edE`3mkA3h z;S1N$-`u!y!(l*t>(*nu9y3p$S+*vD<|dH_B~vw$3qM2Dq`Dk;n2QP!4WL(WX8zk{ zENCTa50vLFmb=BJLS<}>^2X$irfAS^4xnvqbUT&3Cesxm48csqcdLfxl}!fFQ5m@U=8ZTnWEZ zx3YSIuoQFu{u*v+S1srwDH*-Bi9sMjIe-575i}Un$uDgZHXVP0SavLOZ+vDR|L31S zB7mwQ-mcqi=>7Tg=TTG;3}3`2aD$c}pmvnNZJh)N)=#Y;`j`y4jSvi^4#-ZOs`!;= zSh@~Ok`fdK+Jc+wMAJOx-I-!|cQuFtod_hBmX_D9UpGNMpO|l>jd*MSee<$uu#Tkt z3^r$LA~DtGxiJZ4fTcYK>{N$LJZRqB)HIquKdXc~;8?1Xk&)4kvzf{#?rndr7lc^c z2Q5W2!=icMk7)oV0({u(DAc(|9~MM?1{Cm(pt3G&4*UQy9>?!)4Fe|>6<>zL+mDYA z@bU4%irqH}!m2We1i}{`YG@6z%!@^FWD1Q$@XWJ>1}qdP*EfcK#q+>v+NjGwLF@Aw4YGODNN0z*9650XWij&pA$ zQ|5`}G0;ErbF+{HN;%I>Lq+rlMZaH4O4MQKEGV9LZ{F~i&VeS{gz^UJOD||DO*69u zK!i|}#-D!O9D{-yWW`(U{sB;+AkI;8MGz?th^mh7MT`v2IdS2sKDSO(wiS4;#Ki(` zi39Nt=w8FUP#oX#lIPNc;tM}Kl#0hm1!16L1N^4d4NeMh!x*4Ir0|$8__;CAFny3N ztupgOMNvA{V0A^29VMR?-`V7uH#cnx7;y%8yasCJN(gvJM@`n0)1hO0X!Boz8`}Vt z)=S`HWvNO9gs}_tDF#cHEFlssNhdeZ(1U8$+Qc*I&SNG=jvf^R!H2y(UNMCHA1LFI zmjy*KNn)FN%X?y|!efDggW_zNFyBjcvCEan+&$2u(*i ziq}cl#(-(79B=@cF!J^5)BH-wbS+=!iJ?T)5f_Sm`D1UzO;1k7U|p9iU0MaaI0kLe zv^L?C@pvy3Ebs%p#alwFum1FF$G|`WHZS6Wxb^45bT#Lv>+&l#`V*^e`6YKNn4d!O zj|j@w`uO}}^h4)9fgL(IS^zd5wM_{Ie6iZl({iO_MdPTIon6g(%bc)^$8!AuKZ>b? zB|`ZLRnbcQ@yU5vdO?$ujtU_X#OGJy{0(Kzqp0Y|bsDd`vq^3!OJQUz0SB)VkU}H; z@C{pYJ?{#X54>k#!2)x30Tngrf1n-i8!*WQ@a5Cb zw!0sp=!G2F03|A}-(k2Valg~ZTd@A;rf@rx@c|UEjK@PI9W0jf?AJnpT19NL`EJyC zIM>D~6HaGZg%LGx7@Cxn1fJ5~v01^^HjR+J+x+ar$jC@#vELRufM;!#sFkJbZHVEZ z3yVWdr-kZ8D4Qmg5jS!}al#wq7|0zF02ld`B-ptnB2cBpV&|a?5&-UeG7k#5ALSJe zO~1Bj?rIzl6_Ca0h|^%fj)7mOL?jy8tIB~d`l;QbN<_qh-IQw=vjBDTW>E7I4RpRX9a z*SfXHo`fvf7>3_w2-9~#m+k-{1k-yOMfJG*_&~j0 zzTU-KT>D0G)Zz_F?$J#GTW#0%er=-G(?duH?99+z=Psi>qDybsXU$)Ni;AT67&o^u zc?Wozr}&EZ_?Zi7FMqF>KibuH2^`bT`H$Z%hpZ%u%_4FRx0&Sfl^{1z zQyD|Z8aFS1NQZh2>h}vCN4@jstMffPLS#HzT83~GRk4q_p>g~t?h$`N02G2f30=7; zFnui)nah_hRYTGr#c7F-pUceLOKygG)tmmi<=z>;Y@L^8K9pIH2V~>$=`aIEzh+xH z<9dHrc+f%Ay|pOzHB$5$M~)m3*tJU;$%+zII*^5=wCPFT7I)@gc8kB3uY_&SBNV$5 z+xxNX(98G0B*GNI{wM_RRrzbRP+mW$A$^9U`mS*1pXUw}?7j5*PnNZXjbr(L%Hx8! z_*Z2Azu$qD_dn?t`p>OoaB4Q2zRyoVQEwX?C9_ND^DoI4{k^CELA%m_?g9Qoj;R0P zkFfqQ^xbW#Rnd;&Byu`2 zD!;ZDt>onmjeWzaR&T3(Slj#wbOYpyYGe#MI8#I~lpjEK;>;Y(p#KZ;L3&%?_p}Tf z;MNImgyyFat9k^W-w*yTx2fG_`yZp$HZZZK*RbiUUzfAhOwbAg%q@BI=FK^$0|zd_ z7;?|}8tWsigzzRo5D#9ze$80|M~|S8P_n!L`32c7x%js|f*vgkYewx{3+e+4z8E5kb0&%SWWj((Jl(##ENGBS7u zAJJq4jlm2B{mb&L&0AZ8YDGQI)d?;D2GWxH8?Tk=PTve+y}1AYbxLIueKWS+du=P4 zks^%&vz^;-igVrL05J~=RTWh9?!>*U7l{ZKY|7fA7_1DgO>Mge=rTKCjeom1>ycjY zArV@6Hl4?7@Fmo|SdaTj$H1{~B!6y9D0gP01bV06L)Q@oqp~60l$cynz)1X;J;VAO z9=;&qLVf*(TcSsof7fUx2C}b3P35fwau7gVMcR@b{@dg}s5kDZ-#@iLlL(Jb2Ty$X z0qWW9j0^|+0KeG3_wepYJJ$Q%77%g0IRt0cgMa-4F546Sf{~zS`%rV$;4q|yZ)UxC z=+yNqL!db|!VWE(?UtFp15%NkY4Cp;9WiIUL|^Or6$=!aqc0cp_JGbJofBf#PsDZu zA=TX%*0TkQNQn8BUVh28*s5Hi3-A#pmkaprbacT6OXoseBT}a!!!r5TEsFYvA*^?g zD3My+d~>~c91y*r){{rrrN^L<$@)fR1PG=iWJ!a6zWPMg6$>18J6s=#T}Oee<5V7B zr&j@h^y2QYXmJ!@SnF+YX~Gfs;awK%WF6DUx-vw_1}2gVfIlXU&sc~H#Wu^76CBKy z{q@_oDxhDHTrJka4HfRoU4B#8Hv-uTq~6TP zm~dOkx|%O$-zt|cU$P{tT|&>>Oe~0`vFxpXzoW+W%R>O({^FJ`32`j;>Jsx_*rPL` zx1rP+XzXCUtSQ6A{np=Eeg1GSdlo>TCNNY=%BsKL}LwxW*x zpb8jz0K~krF5?4?3JdV;SlEZ~aCHn0CPJ_TFzviGduzS8B2)sLpA1w?8oADPP`&H& z=AgOsRYm)}yDP8>#TAaWJc!R4=m04ld+9rCai|5{i35lekdQb{7z;ZDcoqhDM*e|u zG1L+&aQZMUA@&wa6ur{R%H$J-TlLrtep0{LPmyjBY(xzA4b!o9$ec8er2 z5fPCZyKN^EZi6|n%%6>dM)3~9DJpAUeUIVp?q~og2{X-DRsQAN64mF(=(cj@N`7$K zu+a&Ec7a|)PN?9Oo{a>_ttg}57<;!ZdF$T2S_bd# zs-QAFtF1l!5w=%_nh3_Pkk(HhJa}-O@cFB#l*#MFP@Nj-6xAzSh5(=rk%@NYsP~#M zT+>rO{qz%PAMiZ*2u%c=&feZwoT`2>AC;heY7$QEc7kXJZv;^SZsdxI1@TT!Eo>|& z7gvI=$E@hD(o%Fb>U+bMWg%EaL*xs`u{A+qh$|OLwgM!ESPJcNL4lA9d6=^%Zrx6a>h;P7m=kFC5s)z!(0R|UyEBKWH2@X6@s->hed!) z!t8D+1c+4gPfkvz8v$Jg3_)q0JDD=z49q9VC6sfl1W&T-gc@8QMJ54omq z`uNnr5-}vv`3w#^Vz*#U^zU=l^TrVlx9NOm41Of;WMUwlSvU%NA*I#gJCDNJYc#s) z{cRX-#)eyUMNMnfaZI4_8Cgw#-U_1c+cUk}kT_}Gh1@4W+{JH59zP6KJ z&f$!yA(p~L5%JJ{+W6YFYX%*U>BfKIxSQ17L#B-0p(hZrCeF$`2TQq}{MKDo_veaD zu(%FA`Ow;`i-S^y12BeVInMG?&*I^0o}4S#12Y#wffBBRN%^Bc_+lZ7fVruIPSgo! z6I2>`{B<3I@HyVxCg?R}`yp8lQCG-kwC3wbCw3^rAjcjp`RP-P|gbe_S+$1|4`uuo6N%nUIi*) zU>JtA%7iVVf73pm1sBXgZScjSzQpPEFkJv=sqo9@*vO%TyIuC-N$Q3D!0m_L@5`tR_u__v@$q8SR+Jsnm zC64$KjfS;k@ohjZX@DRkw1SCB(bbedB%PgX_aSem8>cYL0*I;LCw%#Gzr0#{I_<(8 z1fPuz@*9)s9VC}LNL7@%XDsSdB2jo&?&kqpOkNm7nsvK=zXX>zckMhDW4t}9>T8s! zwj&k;ab7D>G$ImD8@Su(7J0{hJL2^*U^N+DRIgUy;P!rLD*(F$HW};^!}pM^EQzK> zHxLXSFekh2+qcIcA{&E;diP*=H&~8zg%Kex)id|FDzSrcI0C&O4Cz|$0bx&qE3#CX z_5=$Hg>KVU3xGZHc9Q0ZZSCTXQxYZF&u1IT;0QB?=H(lKi%iHB7t0U#X4dZ0iv@$N z`K@^hfWlH8$Hiq^64h=Z`ozQK*#j!TmybARX{Hla*m@}M{U(EBJ{%ibO1TSA(x?S zNF)m&1>Ihj;d~le#Wq;CM#jc+O?9Co5{SCDQ#ylmtW}v7KYd5?|YInLHjdUpX5GTXGv)Hj9GpGHzKk zrHBYT27cERtj<5kE`A1tPTc1J1gxm=* z&%rUNNYs4$+kIH5OP+GqEsxs4?n+;gv;wFTZh2 z{fZCw$C5_}&Yx#WY6|XV3MmK%4F&S>-w=VfPf{`-gduw}p2{dDDJEv?zJ2?8k*n$= z7=FZ?yDaZjNjxo@exc%(bg9HC;A3?Jfm}pkfmdTFao}`#m94YbRS3*aG|mgLN^nl4 z<-<*FVp#?5iC9L$5>%pGsQ3-sn}mFR3nJhgA~}-@|5GdypP*WlH$VW~RnX zqX;>I%QFU2puCJSCcy!kCc)|4h1kV(OoG-Co zMhtNfK}%spb;>KUUSa!*r2i*!COhh<7k>!CRxsOeih}Y0gTg3j?~0QgA^Ld7Bn3UM%T- z8(OsqQ85mzh>pk2M(V7Jzv~k zUH!>;d^Hb`yHZt}WvkEKku~UH;krkq;o26$0+&4UmihF|A^c6_7h>cN&lM%`w0QQP zPF3x#j7$goBqa()4Y<41q3Yn5cH*hjLiY@P27NreLJDs{OF{(+A`l~M?Qb6FDMD%l z&QS!5S$VNcDjMrpKU^NP0u3S+ko`%_0I=R#(71TfBI0qg(3(;(f)&1i4EYYcCY@kJ zLaIEeO9r~&u(1hC7}xSI)wi154uTi*gXi&k+XX@E;Q_XX}?;K{H^o6cjiiiHYdO71UMC5mIYZ|X~+cVPdJ=1%=Mtjft))i=^( zn$!7m&s?WE0|bVw2w$_kxN@(})V39EPNEh~S_rzTu)s-Aep-m9J^`_%7R+o)_}^W2 zl{-)ugUPOh1mb`D_9JR>fj<&-@99*-n+Wi!;e{^tUp^g-+ChZvi%ne*c`mJAvnGag zr&Lo`?er-xa$8T~n>Uf!vhmulS&!hD8L_RW3$B$ml|#l1dHI~=%k*SynY_JAvT)g- z&QE=MdBqtgp&IH8Q~O?B!l}nV?59!tybly)z3s7R+eOWgzdp{-6t1_t01qhvJ-2!) z#X$k|fV~H!6gX(>;GqfPop(aeN&}+(w-x@Gvs3BuNLURJaSY$zztg+uZ_;DMTAm7XU@`JqqekgN zg$4MXKrTL+xk*{LFN?j_{r&72*q-C6WHPEfV`*UjFNNTeEJ$f}s7liy7BeWeNjMB` z2W=$3gKz`eCGU4%syhL6DWW5o=DGEapeEgo229OIN!r~0#eBy=y0)K zv(`?jtinG6P)UG~k8k(C95Bfq`=O?N+Jn@`;Ii`vdhI2q|InlLDCVfVLT(Na-n`B9 zZ_4Ijq+}1$ni>H_tv29!l)`!hb@*ttsfLq=!nIiTCnzaX*Uc6O=&6 z3Ec0LLh7r3I_q5BJ_r5)oj;l|0jWcRAgqUgdm_CVmGi%HUCjqGkZmB4ojS$(8f+6g znh%O^-n@A!i;-sAaIxu|b^z-gC%63fG}q$0{!enbe^bfpk|vMixYld4_E2Bq|EJy^ zuZ#ApFAZ4RHhFY*b_F&Y&^hz*4xeo)8Ty51?f=@$`u~^oi+@*F`Tyv_|8mJ2ffA_4 zKOi7rW@d&A(t~J%3R6Hev~tawtzh=1rl;dhr5a8H%X2r3q9R34s*Rl;EHeJ81Ko%x z=nH&&=+Gf@0gwkaJ`W{$3{ViTbASyh^YgRtRTIzzDfN$DS`AdJ==}(2@%i+}hMp)j zd3pIb-92>9BIEN5Px02z4Mf4p391vB04tmo@Qq zgT84w7{$k^nD9SXjnvWDk^8l9JuH*QE@s2N#I$#C20Z@ql8Z&%MpVJZLjVIHH6BJ^ zuNyZiNW4eGi%Sr>w6|}UM|~G}_Ib*64&EeSkG=ZA@FOdOKA;|yIOf+cU&=GB+ee2z z=Fgy|w>Qc1QiWHr9X|2?P3%fshT`u{!dy-Snt2(;hVwHNSSixB)VSTxuYvm9uVbYLv6xv}qH> zil4~u1k@Ey&C&3Ci1pHBq){!tg`y$*IKOHTu^f(j1H>(C?FT0q-%X<0TLseA(H z2#HO~2M0_28|Zr0-`c)^E1`Vk4Gqu?+0VFP=!JD*-WX*7%{PD#3s_+T758YLy$1O{ z8bzSC?_Hoo9HV$pAk<+*HZBw}3b4#++Zni&+u7OqLv|o?3x89xGrce=G0h3ef{7T zB(-N-ixCw#SFi5=LOAU9EGUL_y&`sy#r$v3U*2F0tO9!N0Wc5-HAJEoF_iU>bpiG! zc9^VOz=U;ZGYHMLP=koY3|-2(bpwN4Edu|3gi4K(weM9kE;KDI0BFr(Uf0`U2ak5! zX1qUNxP~l%;4(GA6`*}(TTP6LEJ=Q_w}2ZFn4RuVhUw>A;Out1j!0)f!vR?)6i_-R zVrCy7q37jz);?{g$_NH17=J!0g<$!s!)aT|$;rnM=*n)*w3a0R21hAhET}S494qZy z!S>hvC6pI~lA}I=zs{GInyg`by`|!<>NqUY(>3l<#D zD@%=E;L)FYHdS}L^Dp7C;jdpf*&HlkcN{TBlRD$gsHvQDc-Y~v1FGdL1f^L!kdDkg zD)L|C?k}fQm3(ujh?(I?g8eKC{CraOD`Qh6ndS4REM$`oZ5Q)!E~F z;CyQ13n=c~xszp851CJnz)#Y=gbFfOf*(UKrebg2o`KCm!H$?Tj&=786Itpr@V#IsYgHIfM%z4jsjdNoQJ~F)o*E2sGMswnQ$Jep^MCLdO}*v!npAaTafDC=9dbYEY%nEm5IBivUmo+7`d%}!qia1MOA z7MHWy^%5G+WKd)ItAc_~4C_FuJOoCr91%%9?ScwexkH$M@y^X{X9f(J0q*m&4sIB# zB5|?Hd&#QpF)+*84Yx>XHyXK=E1ZBgtQyU2%enR{V_)U%hNcH$^5p#GmvD+=aG(Kl zD5D4@WeS#!&RdN%2nElLFdRuF6S*%TR)9&xu#^rr-V&(UB=L}*2D&s0Q$HZzM54+1 zW1~EJxd=jFXv`?QeZ)Kj!b1l~=0O2y?3C|;*m^KyqugaU*P+#p3O7fqA+b$q%3+75 zp$H8(ltCLBZs@X5GKxiKXGBnd-R8e;+)$C1f6tu}dnO|Zm&UEmtrZ6J4R5T`*B-5f zZZJ8}cq(udgYBT%MtJeGI`Fo!v3|#lC@H zWwbM>QJiEbNH#q0$3G)XVQ z?O>ojf97+j`~3q}rH=cR1kJJssG=4^!|q6yE#=8CL5#8^QT07AS65tGu+SFs2|TJV zJ{AcAx=J{e3P}8Id$STWF(@5r7uC~pnlf2~qa2mJ#+gQ?Z{=C`iYK~{O;^r;j4eTX zQ7s|@x5;g%xu-C{fEx8jYz5R?x8KpqKH#DzcuxmccpHDqE+GQ!l3byT z{b2_vi7@D1un)LnVcfjnIa0qv^Bk}A6&8uXvu^(5Bj%ImRhxYA6X5=9W$gP{tRUOO z<^R$IE4RsZ-{Sq6(+WtGzhqV!mP%>ULlr2oe@AL>(b zeBup{N@ixJSrQd72kei3_`Z1U%i}+0`P@fC^)X+h;J{LU{1w~emQ2JE9Q%}PZ}}r= z&g#HCF_r{e67qi12vnOI7%imP6~NR+4*MKmTg`2hw4U3TZ8g+p;Vn#22p8^dbELk+ z<)KZ5#l2ky^#C3-S3J$y1_w)@dzuwYc@DhBuUF4K)3ofq{1v3aVvYXIDa6mA^Bh*P zJMrU9(4^}DWmW|NF<0C_K>6wj7UiXZ7oQZbz!AQyyO9+S_B>6@WcG|xxJmc zVFo_8I0RVC@u8%W`|X3|DYpxbKQ;}{^do_Jd2pau<8H8;Yt#9^E8GodJ+k5Vrg>yg z;z;y??I30P%cbqI9Bk_IXaSpL}Ev-0@5!{6eH>rCT^+^P{kYS@w*brc^?_jq~d>WA#-7^?6N736y&#@lO^O?kRJHLDjvIbjNTK!pkrTQw9^WSioa z&AaFFr2t!~`Jg88j3$*VXj7@dR3F4LS&uJtgw-hME8>W@#04(=v zZ+QlRX|CPQWWu`9j`k(`I?NQ{p7olVn!7QeM$cOq3nAkB9^ZMu9J+=nD#m?q^$Qo? zM~SP$CZF$Hv2l55GnGII4lvjGvY+^+OcNRiGI+fRVtOLkqZ< z^PI#1^2gmKceFX@#sYf|!MH1|=tTgoB3KdgB|%gH5nc zKq1ita)A1ue4$Uuz=ypRSGVr@~pT4Wrum3qf)g`VfYwo= z^7GTE3!054cJhb#8}K~QyaDheXt<;CYV6(3*jRECF`ZDk*_pTIHMe=@fBdLL(*pqa zTt1Dpw-F@+p;t2468=^iu2V2&KQ}{u-QpHQFk&>S5OYn=S*Bu?PIIPpLCa=PhGen> zz~(8hc(H`aC1kZ2RIt8#=dFhS?vcZliNuP>AHMMmd;jkK`9`pqlB0K;u7|GtTK;Y* z`&`@0`yvKj)^>KHu)-Lkc!@&D*o;Wc?$EkWr0>mY-90#>qKnzNcA=?81We;*kGYvH z2tgPtg<(kN;2<(YrD-N+(R2ZB2@CqwDvdE2aACIWG5c6<)|^%IK1jTb#!I1%$*68) z8}9&=CPSPj`gPP7ZSCH;%YW^&?YWIOfRM@Rmq+FZLfKrp$F%nS-K{FGKn=ZVkwYwW zLQPbP==mPJs{)j3WFqr{{xa}RbV+p_+ZLsQE7=V$+CpX15>YUBdGncrRgJ)g17fzk5HQ`ZJ6so^Ya%hbFdx3=V>!YVW8uI*Jeh*SGif z(g-jGP?Ybn3fv85aPooNDWO->(_q=9Vq{;-XknRv1r$(W+(a49nx%kI1Z2Al`5aIj z?2)-THpIdAJ)UpSTm(l7Li~NOuxnSZUadw%pc@;*}iN|U8^T= zzj^_iV;bt5qo9W=rNXZfFtBq=36K!^1&Q665L>i_QyEKC4Z4S>LY9tEvf0(<~w zpoK{$4Wk~Do}1L8ZYt2FE|>h9ZQ+Z-tSgrB1g55YgWZtY-BrXm$Wg~{1@TG?A_I5w>F>3juS%4_yQfy_AV$?BZgRl(#z_2%u z?UPDDwE^3Uh3KxJF10wEb267AW$5YFJu^Q;2Q+LJc;YSw!BSc2dlyX*=!=U({D7TF z6B&`%XBQ?cGh`x0I>C%ce@v^O35>`F@z77m;tG0&COM&5xcr%Udy z5Rz9e7IY8VtzCkE4dQ8G{vIHN=oAfaM*ZSjnzOPAN!Sfaxuy5 zA%Fu6U4je(aL95y@a`C>rH%_#gQpvCTm1d^JwzEoNADaQG!JzSdN@cEYsoB)-VyRy z5xL`;MPqnTdT&M6qyA~~I{>^frxAbRK_e40jwVrbqVMA86}kY#h}jp+IzsgH@A^#28zoE8t|Kf*&UEi*c$g zE$(nrQMh9bG_8bFLDaaCkA{2UG*uHy#!w+s1Z#Gt6FetW`-8APgU?dN_+p?1sO_tW zoJO;bI>HWdZcH%{W@Kci?#xmQFax-K4$-5t+!84TfSfg44l!I696w1v$joB0p#_i~ z;;83=Ff3;5k~)C2O@C&2Sh^*r|R z7A_os8S+ACr8_zzQr1kXG-hKNXAg-bKprJU%EjiC+ zgGMbTa10!6ZRfj{Z~)85h8{sL$$-aBIA;0isWE`TEN(CZHv(u(q1A07 z#O;CbQ8b!UeL<1A&pO&6`%M4Sdb5#xA4x$;IeO( zy88Qr`lNlMSD_Jq=aHc7!TIswr=Y)q{ch7tchCxPP|V@_G`s@f=pD~ynk@`JdAwFn^GmxmyHU|64fD{}$c-p_{Wh;!#bTfhq#Om;h*Hpv_u zkKQlasl(C)ls707>VOz1$<4{3(U!(9E-fI9E*(Qth$y1T)tHCTzh?r1w(&o7-x~Zd z+@@zd;SM$^Au%4Yy$aDe9RsdOwZ}G&-^l{&g(ewoT3y)kZQ=H$mVcVTh4Scrap?WW zG~$Es8i zQR)_jogzC~FOM2c)R4KUDT>Aipj<15jzOX&b9uZ)H>Mll(U22I@+Ri-UCF9W)iSJr zG_CMswAYDB+&lO=qB!#>7qB(S0Dyn`<37X_s5Po?($t4U_FrmG^|konF7PkO^rV%^ zHe$%M8i8+8W3h!a>6@YthP~YBnk#~agGfWkho%j}^V?x1mBdHrlCU-gSrXKkWwy?f z9OyQk2r&ZLkN(35zLpn#WKilwztLm3&0|oMIitU642?$di2DDi-96F1(VqAS@@|9f-3r$WCN)3NwcOe4%BMz7LL4k+ z0|(m%{vhG``3EqfzIcgyP*@p>zHG?m%cV7XqZrS70S6C8Tm@7NP?FWrJV&SO z?}{Pw=xVl0^OmUmsTmlt?jPn#0rbN`9IMw}$Fl+O7zc!!zRFVEW>E5AyvLDyKNzz) z-W3jC%JyPj98E^_pP`^RKj?vsB)>KCf4%e~C^h=@pMStVzY8>Zvl^TK3uD6G%`a^A WM33T4<~4dt#u3HCNr!&F{C@z+ACZ0l literal 0 HcmV?d00001 From a72ba3164bbd429e42e0f8db22680bab7ec6778b Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Wed, 11 Oct 2023 21:26:08 +0300 Subject: [PATCH 50/52] Fixed Dockerfile condition --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f0d120e..6c1da6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,10 @@ ARG ENVIRONMENT=cpu # Conditionally set the base image based on the environment -FROM ${ENVIRONMENT=cpu?'python:3.8-slim':'nvcr.io/nvidia/tensorrt:23.08-py3'} +FROM python:3.8-slim as base_cpu +FROM nvcr.io/nvidia/tensorrt:23.08-py3 as base_gpu + +FROM base_${ENVIRONMENT} # Install required system packages conditionally RUN apt-get update && apt-get install -y python3-pip git && \ From bb99ebf248a7ea8aa1addff42b99f031a4624e07 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Wed, 11 Oct 2023 21:39:16 +0300 Subject: [PATCH 51/52] Revert Dockerfile to mention GPU image explicitly --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c1da6b..9050db0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -# Argument to determine environment: cpu or gpu (default is cpu) -ARG ENVIRONMENT=cpu +# Argument for base image. Default is a neutral Python image. +ARG BASE_IMAGE=python:3.8-slim -# Conditionally set the base image based on the environment -FROM python:3.8-slim as base_cpu -FROM nvcr.io/nvidia/tensorrt:23.08-py3 as base_gpu +# Use the base image specified by the BASE_IMAGE argument +FROM $BASE_IMAGE -FROM base_${ENVIRONMENT} +# Argument to determine environment: cpu or gpu (default is cpu) +ARG ENVIRONMENT=cpu # Install required system packages conditionally RUN apt-get update && apt-get install -y python3-pip git && \ From 6349fa84e1aa6fe29c0ad357d24f9469285f1f3b Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Wed, 11 Oct 2023 21:46:14 +0300 Subject: [PATCH 52/52] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d9a1a1b..4aca697 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Depending on target environment (CPU or GPU), choose a different base image. 2. **GPU Deployment**: If your system has GPU support and you have NVIDIA Docker runtime installed, you can use the TensorRT base image to leverage GPU acceleration. ```bash - docker build --build-arg ENVIRONMENT=gpu -t my_image_gpu . + docker build --build-arg ENVIRONMENT=gpu --build-arg BASE_IMAGE=nvcr.io/nvidia/tensorrt:23.08-py3 -t my_project_image_gpu . ``` ### Running the Docker Container @@ -68,7 +68,7 @@ Depending on target environment (CPU or GPU), choose a different base image. 2. **GPU Version**: ```bash - docker run --gpus all -it --rm -v my_image_gpu + docker run --gpus all -it --rm my_image_gpu ``` ### Run the Script inside the Container