# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license """ Benchmark YOLO model formats for speed and accuracy. Usage: from ultralytics.utils.benchmarks import ProfileModels, benchmark ProfileModels(['yolo26n.yaml', 'yolov8s.yaml']).run() benchmark(model='yolo26n.pt', imgsz=160) Format | `format=argument` | Model --- | --- | --- PyTorch | - | yolo26n.pt TorchScript | `torchscript` | yolo26n.torchscript ONNX | `onnx` | yolo26n.onnx OpenVINO | `openvino` | yolo26n_openvino_model/ TensorRT | `engine` | yolo26n.engine CoreML | `coreml` | yolo26n.mlpackage Core AI | `coreai` | yolo26n.aimodel TensorFlow SavedModel | `saved_model` | yolo26n_saved_model/ TensorFlow GraphDef | `pb` | yolo26n.pb TensorFlow Edge TPU | `edgetpu` | yolo26n_edgetpu.tflite LiteRT | `litert` | yolo26n.tflite PaddlePaddle | `paddle` | yolo26n_paddle_model/ MNN | `mnn` | yolo26n.mnn NCNN | `ncnn` | yolo26n_ncnn_model/ IMX | `imx` | yolo26n_imx_model/ RKNN | `rknn` | yolo26n_rknn_model/ ExecuTorch | `executorch` | yolo26n_executorch_model/ Axelera AI | `axelera` | yolo26n_axelera_model/ DEEPX | `deepx` | yolo26n_deepx_model/ Qualcomm QNN | `qnn` | yolo26n_qnn.onnx Hailo | `hailo` | yolo26n_hailo_model/ Huawei Ascend | `ascend` | yolo26n_ascend_model/ AMD Xilinx | `xilinx` | yolo26n_xilinx_model/ """ from __future__ import annotations import glob import platform import shutil import time from copy import deepcopy from importlib.util import find_spec from pathlib import Path import numpy as np import torch.cuda from ultralytics import RTDETR, YOLO, YOLOWorld from ultralytics.cfg import TASK2DATA, TASK2METRIC from ultralytics.engine.exporter import export_formats from ultralytics.nn.modules import Segment26 from ultralytics.utils import ( ARM64, ASSETS, IS_DOCKER, IS_JETSON, LINUX, LOGGER, MACOS, TQDM, WEIGHTS_DIR, is_github_action_running, ) from ultralytics.utils.checks import IS_PYTHON_MINIMUM_3_13, check_imgsz, check_yolo, is_rockchip from ultralytics.utils.files import file_size from ultralytics.utils.torch_utils import get_cpu_info, select_device def benchmark( model=WEIGHTS_DIR / "yolo26n.pt", data=None, imgsz=160, quantize=None, device="cpu", verbose=False, eps=1e-3, format="", **kwargs, ): """Benchmark a YOLO model across different formats for speed and accuracy. Args: model (str | Path | YOLO): Path to the model file or directory, or a loaded YOLO model instance. data (str | None): Dataset to evaluate on, inherited from TASK2DATA if not passed. imgsz (int | list[int]): Square image size for the benchmark. quantize (int | str | None): Requested precision: 16 (FP16), 8 (INT8), or None/32 (FP32). Exported rows apply it at export, where a format may reject an explicit 32 or fall back to the precision it requires; the native PyTorch row is not exported and only 16 affects it, selecting FP16 inference. Each format then runs inference at its own runtime precision. device (str): Device to run the benchmark on, e.g. 'cpu', 'cuda:0' or 'mps'. verbose (bool | float): If True or a float, raise on non-assertion benchmark failures; a float also asserts that every successful format's metric exceeds this floor value. eps (float): Epsilon value for divide by zero prevention. format (str): Export format for benchmarking. If not supplied, all formats are benchmarked. **kwargs (Any): Export options; nms selects the same head for native and exported benchmarks. Returns: (polars.DataFrame): A Polars DataFrame of string-formatted benchmark results for each format, including status, file size, metric, inference time, and FPS. Examples: Benchmark a YOLO model with default settings: >>> from ultralytics.utils.benchmarks import benchmark >>> benchmark(model="yolo26n.pt", imgsz=640) """ from ultralytics.cfg import _handle_deprecation kwargs = _handle_deprecation(kwargs) imgsz = check_imgsz(imgsz) assert imgsz[0] == imgsz[1] if isinstance(imgsz, list) else True, "benchmark() only supports square imgsz." import polars as pl # scope for faster 'import ultralytics' pl.Config.set_tbl_cols(-1) # Show all columns pl.Config.set_tbl_rows(-1) # Show all rows pl.Config.set_tbl_width_chars(-1) # No width limit pl.Config.set_tbl_hide_column_data_types(True) # Hide data types pl.Config.set_tbl_hide_dataframe_shape(True) # Hide shape info pl.Config.set_tbl_formatting("ASCII_BORDERS_ONLY_CONDENSED") device = select_device(device, verbose=False) if isinstance(model, (str, Path)): model = YOLO(model) data = data or TASK2DATA[model.task] # task to dataset, i.e. coco8.yaml for task=detect key = TASK2METRIC[model.task] # task to metric, i.e. metrics/mAP50-95(B) for task=detect y = [] t0 = time.time() format_arg = format.lower() if format_arg: formats = frozenset(export_formats()["Argument"]) assert format_arg in formats, f"Expected format to be one of {formats}, but got '{format_arg}'." for name, export_format, suffix, cpu, gpu, valid_args, _ in zip(*export_formats().values()): emoji, filename = "❌", None # export defaults try: if format_arg and format_arg != export_format: continue if IS_PYTHON_MINIMUM_3_13 and not format_arg and export_format in {"saved_model", "pb", "edgetpu"}: continue # Checks if export_format == "pb": assert model.task != "obb", "TensorFlow GraphDef not supported for OBB task" elif export_format == "edgetpu": assert LINUX and not ARM64, "Edge TPU export only supported on non-aarch64 Linux" assert shutil.which("edgetpu_compiler"), "Edge TPU benchmark requires edgetpu_compiler" elif export_format == "coreml": assert MACOS or (LINUX and not ARM64), "CoreML export only supported on macOS and non-aarch64 Linux" # coremltools deadlocks after OpenVINO on macOS Python 3.13 (conflicting OpenMP runtimes); CoreML # is still benchmarked on non-aarch64 Linux Python 3.13. assert not (MACOS and IS_PYTHON_MINIMUM_3_13), ( "CoreML not benchmarked on macOS Python>=3.13 (coremltools/OpenVINO OpenMP deadlock)" ) if export_format in {"saved_model", "pb", "edgetpu"}: assert not isinstance(model, YOLOWorld), "YOLOWorldv2 TensorFlow exports not supported by onnx2tf yet" if export_format == "paddle": assert not isinstance(model, YOLOWorld), "YOLOWorldv2 Paddle exports not supported yet" assert (LINUX and not IS_JETSON) or MACOS, "Windows and Jetson Paddle exports not supported yet" # PaddlePaddle export works standalone on Python 3.13 but its native protobuf clashes with the # protobuf>=6.31.1 that TensorFlow loads earlier in this shared benchmark process, causing a segfault. assert not IS_PYTHON_MINIMUM_3_13, ( "PaddlePaddle not benchmarked on Python>=3.13 (protobuf ABI conflict with TensorFlow)" ) if export_format == "mnn": assert not isinstance(model, YOLOWorld), "YOLOWorldv2 MNN exports not supported yet" # MNN export works standalone on Python 3.13 but its ONNX-parsing protobuf clashes with the # protobuf>=6.31.1 that TensorFlow loads earlier in this shared benchmark process, aborting the run. assert not IS_PYTHON_MINIMUM_3_13, ( "MNN not benchmarked on Python>=3.13 (protobuf ABI conflict with TensorFlow)" ) if export_format == "ncnn": assert not isinstance(model, YOLOWorld), "YOLOWorldv2 NCNN exports not supported yet" if export_format == "imx": assert not isinstance(model, YOLOWorld), "YOLOWorldv2 IMX exports not supported" assert model.task in {"detect", "classify", "pose", "segment"}, ( "IMX export is only supported for detection, classification, pose estimation and segmentation tasks" ) assert "C2f" in model.__str__(), "IMX only supported for YOLOv8n and YOLO11n" if export_format == "rknn": assert not isinstance(model, YOLOWorld), "YOLOWorldv2 RKNN exports not supported yet" assert LINUX, "RKNN only supported on Linux" assert not is_rockchip(), "RKNN Inference only supported on Rockchip devices" if export_format == "executorch": assert not isinstance(model, YOLOWorld), "YOLOWorldv2 ExecuTorch exports not supported yet" if export_format == "axelera": assert not isinstance(model, YOLOWorld), "YOLOWorldv2 Axelera exports not supported" assert LINUX and not (ARM64 and IS_DOCKER), ( "export is only supported on Linux and is not supported on ARM64 Docker." ) assert not (model.task == "segment" and any(isinstance(m, Segment26) for m in model.model.modules())), ( "Axelera export does not currently support YOLO26 segmentation models" ) if export_format == "xilinx": assert LINUX, "AMD Xilinx export only supported on Linux" assert find_spec("quark"), "AMD Xilinx benchmark requires amd-quark" if export_format == "litert": assert MACOS or (LINUX and not ARM64), "LiteRT benchmark only supported on Linux x86 and macOS" # benchmark() deadlocks on the ai-edge-litert/TensorFlow abseil mutex (RAW: Lock blocking) on macOS CI # when litert runs after other TF-based formats in the shared process; still benchmarked locally. assert not (MACOS and is_github_action_running()), ( "LiteRT not benchmarked on macOS CI (ai-edge-litert/TF abseil mutex deadlock)" ) if "cpu" in device.type: assert cpu, "inference not supported on CPU" if "cuda" in device.type: assert gpu, "inference not supported on GPU" # Export if export_format == "-": filename = model.pt_path or model.ckpt_path or model.model_name exported_model = deepcopy(model) # PyTorch format else: export_data = data if "data" in valid_args else None filename = model.export( imgsz=imgsz, format=export_format, quantize=quantize, data=export_data, device=device, verbose=False, **kwargs, ) exported_model = RTDETR(filename) if isinstance(model, RTDETR) else YOLO(filename, task=model.task) assert suffix in str(filename), "export failed" emoji = "❎" # indicates export succeeded # Predict assert model.task != "pose" or export_format != "pb", "GraphDef Pose inference is not supported" assert export_format != "edgetpu", "inference not supported" assert export_format != "coreml" or platform.system() == "Darwin", "inference requires macOS>=10.13" assert export_format != "coreai" or MACOS, "inference requires macOS" assert export_format != "axelera", "inference only supported on Axelera hardware" exported_model.predict( ASSETS / "bus.jpg", imgsz=imgsz, device=device, quantize=quantize, nms=kwargs.get("nms"), verbose=False, ) # Validate results = exported_model.val( data=data, batch=1, imgsz=imgsz, rect=False, plots=False, device=device, quantize=quantize, nms=kwargs.get("nms"), verbose=False, conf=0.001, # all the pre-set benchmark mAP values are based on conf=0.001 ) metric, speed = results.results_dict[key], results.speed["inference"] fps = round(1000 / (speed + eps), 2) # frames per second y.append([name, "✅", round(file_size(filename), 1), round(metric, 4), round(speed, 2), fps]) except Exception as e: if verbose: assert type(e) is AssertionError, f"Benchmark failure for {name}: {e}" LOGGER.error(f"Benchmark failure for {name}: {e}") y.append([name, emoji, round(file_size(filename), 1), None, None, None]) # mAP, t_inference # Print results check_yolo(device=device) # print system info df = pl.DataFrame(y, schema=["Format", "Status❔", "Size (MB)", key, "Inference time (ms/im)", "FPS"], orient="row") df = df.with_row_index(" ", offset=1) # add index info df_display = df.with_columns(pl.all().cast(pl.String).fill_null("-")) name = model.model_name dt = time.time() - t0 legend = "Benchmarks legend: - ✅ Success - ❎ Export passed but validation failed - ❌️ Export failed" s = f"\nBenchmarks complete for {name} on {data} at imgsz={imgsz} ({dt:.2f}s)\n{legend}\n{df_display}\n" LOGGER.info(s) with open("benchmarks.log", "a", errors="ignore", encoding="utf-8") as f: f.write(s) if verbose and isinstance(verbose, float): metrics = df[key].to_numpy() # values to compare to floor floor = verbose # minimum metric floor to pass, i.e. = 0.29 mAP for YOLOv5n assert all(x > floor for x in metrics if not np.isnan(x)), f"Benchmark failure: metric(s) < floor {floor}" return df_display class ProfileModels: """ProfileModels class for profiling different models on ONNX and TensorRT. This class profiles the performance of different models, returning results such as model speed and FLOPs. Attributes: paths (list[str]): Paths of the models to profile. num_timed_runs (int): Minimum number of timed runs for ONNX profiling; TensorRT uses 50x this minimum. num_warmup_runs (int): Number of warmup runs before profiling. min_time (float): Minimum number of seconds to profile for. imgsz (int): Image size used in the models. quantize (int | str | None): Export precision for TensorRT profiling, e.g. 16 (FP16) or 8 (INT8). trt (bool): Flag to indicate whether to profile using TensorRT. device (torch.device): Device used for profiling. Methods: run: Profile YOLO models for speed across ONNX and TensorRT formats. get_files: Get all relevant model files. get_onnx_model_info: Return placeholder metadata for an ONNX model. iterative_sigma_clipping: Apply sigma clipping to remove outliers. profile_tensorrt_model: Profile a TensorRT model. check_dynamic: Check whether an ONNX input shape is dynamic. profile_onnx_model: Profile an ONNX model. generate_table_row: Generate a table row with model metrics. generate_results_dict: Generate a dictionary of profiling results. print_table: Print a formatted table of results. Examples: Profile models and print results >>> from ultralytics.utils.benchmarks import ProfileModels >>> profiler = ProfileModels(["yolo26n.yaml", "yolov8s.yaml"], imgsz=640) >>> profiler.run() """ def __init__( self, paths: list[str], num_timed_runs: int = 100, num_warmup_runs: int = 10, min_time: float = 60, imgsz: int = 640, quantize: int | str | None = 16, trt: bool = True, device: torch.device | str | None = None, ): """Initialize the ProfileModels class for profiling models. Args: paths (list[str]): List of paths of the models to be profiled. num_timed_runs (int): Minimum number of timed runs for ONNX profiling; TensorRT uses 50x this minimum. num_warmup_runs (int): Number of warmup runs before the actual profiling starts. min_time (float): Minimum time in seconds for profiling a model. imgsz (int): Size of the image used during profiling. quantize (int | str | None): Export precision for TensorRT profiling, e.g. 16 (FP16, default) or 8 (INT8). trt (bool): Flag to indicate whether to profile using TensorRT. device (torch.device | str | None): Device used for profiling. If None, it is determined automatically. Notes: quantize applies only to the TensorRT profiling export; ONNX profiling stays FP32 (FP16 is slower on CPU). """ self.paths = paths self.num_timed_runs = num_timed_runs self.num_warmup_runs = num_warmup_runs self.min_time = min_time self.imgsz = imgsz self.quantize = quantize self.trt = trt # run TensorRT profiling self.device = select_device(device, verbose=False) def run(self): """Profile YOLO models for speed across ONNX and TensorRT formats. Returns: (list[dict]): List of dictionaries containing profiling results for each model. Examples: Profile models and print results >>> from ultralytics.utils.benchmarks import ProfileModels >>> profiler = ProfileModels(["yolo26n.yaml", "yolo11s.yaml"]) >>> results = profiler.run() """ files = self.get_files() if not files: LOGGER.warning("No matching *.pt or *.onnx files found.") return [] table_rows = [] output = [] for file in files: engine_file = file.with_suffix(".engine") if file.suffix in {".pt", ".yaml", ".yml"}: model = YOLO(str(file)) model.fuse(verbose=False) model_info = model.info(imgsz=self.imgsz) if self.trt and self.device.type != "cpu" and not engine_file.is_file(): engine_file = model.export( format="engine", quantize=self.quantize, imgsz=self.imgsz, device=self.device, verbose=False, ) onnx_file = model.export( format="onnx", imgsz=self.imgsz, device=self.device, verbose=False, ) elif file.suffix == ".onnx": model_info = self.get_onnx_model_info(file) onnx_file = file else: continue t_engine = self.profile_tensorrt_model(str(engine_file)) t_onnx = self.profile_onnx_model(str(onnx_file)) table_rows.append(self.generate_table_row(file.stem, t_onnx, t_engine, model_info)) output.append(self.generate_results_dict(file.stem, t_onnx, t_engine, model_info)) self.print_table(table_rows) return output def get_files(self): """Return a list of paths for all relevant model files given by the user. Returns: (list[Path]): List of Path objects for the model files. """ files = [] for path in self.paths: path = Path(path) if path.is_dir(): extensions = ["*.pt", "*.onnx", "*.yaml"] files.extend([file for ext in extensions for file in glob.glob(str(path / ext))]) elif path.suffix in {".pt", ".yaml", ".yml"}: # add non-existing files.append(str(path)) else: files.extend(glob.glob(str(path))) LOGGER.info(f"Profiling: {sorted(files)}") return [Path(file) for file in sorted(files)] @staticmethod def get_onnx_model_info(onnx_file: str): """Return placeholder ONNX model metadata, since layers, parameters, gradients, and FLOPs are not extracted. Args: onnx_file (str): Path to the ONNX model file (currently unused). Returns: (tuple[float, float, float, float]): Zeros for (num_layers, num_params, num_gradients, num_flops). """ return 0.0, 0.0, 0.0, 0.0 # return (num_layers, num_params, num_gradients, num_flops) @staticmethod def iterative_sigma_clipping(data: np.ndarray, sigma: float = 2, max_iters: int = 3): """Apply iterative sigma clipping to data to remove outliers. Args: data (np.ndarray): Input data array. sigma (float): Number of standard deviations to use for clipping. max_iters (int): Maximum number of iterations for the clipping process. Returns: (np.ndarray): Clipped data array with outliers removed. """ data = np.array(data) for _ in range(max_iters): mean, std = np.mean(data), np.std(data) # Include exact-boundary and zero-variance samples. clipped_data = data[(data >= mean - sigma * std) & (data <= mean + sigma * std)] if len(clipped_data) == len(data): break data = clipped_data return data def profile_tensorrt_model(self, engine_file: str, eps: float = 1e-3): """Profile YOLO model performance with TensorRT, measuring average run time and standard deviation. Args: engine_file (str): Path to the TensorRT engine file. eps (float): Small epsilon value to prevent division by zero. Returns: (tuple[float, float]): Mean and standard deviation of inference time in milliseconds. """ if not self.trt or not Path(engine_file).is_file(): return 0.0, 0.0 # Model and input model = YOLO(engine_file) input_data = np.zeros((self.imgsz, self.imgsz, 3), dtype=np.uint8) # use uint8 for Classify # Warmup runs elapsed = 0.0 for _ in range(3): start_time = time.perf_counter() for _ in range(self.num_warmup_runs): model(input_data, imgsz=self.imgsz, verbose=False) elapsed = time.perf_counter() - start_time # Compute number of runs as higher of min_time or num_timed_runs * 50 num_runs = max(round(self.min_time / (elapsed + eps) * self.num_warmup_runs), self.num_timed_runs * 50) # Timed runs run_times = [] for _ in TQDM(range(num_runs), desc=engine_file): results = model(input_data, imgsz=self.imgsz, verbose=False) run_times.append(results[0].speed["inference"]) # already in milliseconds run_times = self.iterative_sigma_clipping(np.array(run_times), sigma=2, max_iters=3) # sigma clipping return np.mean(run_times), np.std(run_times) @staticmethod def check_dynamic(tensor_shape): """Return True if any dimension of an ONNX input tensor shape is dynamic (non-integer or negative).""" return not all(isinstance(dim, int) and dim >= 0 for dim in tensor_shape) def profile_onnx_model(self, onnx_file: str, eps: float = 1e-3): """Profile an ONNX model, measuring average inference time and standard deviation across multiple runs. Args: onnx_file (str): Path to the ONNX model file. eps (float): Small epsilon value to prevent division by zero. Returns: (tuple[float, float]): Mean and standard deviation of inference time in milliseconds. """ import onnxruntime as ort from ultralytics.nn.backends import ONNXBackend # Session options for consistent benchmarking sess_options = ort.SessionOptions() sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL sess_options.intra_op_num_threads = 8 # Limit the number of threads # Initialize ONNXBackend with CPU device for consistent benchmarking backend = ONNXBackend(onnx_file, device=torch.device("cpu"), fp16=False, session_options=sess_options) # Prepare input data dictionary for multi-input models input_data_dict = {} for input_tensor in backend.session.get_inputs(): input_type = input_tensor.type if self.check_dynamic(input_tensor.shape): if len(input_tensor.shape) != 4 and self.check_dynamic(input_tensor.shape[1:]): raise ValueError(f"Unsupported dynamic shape {input_tensor.shape} of {input_tensor.name}") input_shape = ( (1, 3, self.imgsz, self.imgsz) if len(input_tensor.shape) == 4 else (1, *input_tensor.shape[1:]) ) else: input_shape = input_tensor.shape # Mapping ONNX datatype to numpy datatype if "float16" in input_type: input_dtype = np.float16 elif "float" in input_type: input_dtype = np.float32 elif "double" in input_type: input_dtype = np.float64 elif "int64" in input_type: input_dtype = np.int64 elif "int32" in input_type: input_dtype = np.int32 else: raise ValueError(f"Unsupported ONNX datatype {input_type}") input_data_dict[input_tensor.name] = np.random.rand(*input_shape).astype(input_dtype) # Warmup runs elapsed = 0.0 for _ in range(3): start_time = time.perf_counter() for _ in range(self.num_warmup_runs): backend.forward(input_data_dict) elapsed = time.perf_counter() - start_time # Compute number of runs as higher of min_time or num_timed_runs num_runs = max(round(self.min_time / (elapsed + eps) * self.num_warmup_runs), self.num_timed_runs) # Timed runs run_times = [] for _ in TQDM(range(num_runs), desc=onnx_file): start_time = time.perf_counter() backend.forward(input_data_dict) run_times.append((time.perf_counter() - start_time) * 1000) # Convert to milliseconds run_times = self.iterative_sigma_clipping(np.array(run_times), sigma=2, max_iters=5) # sigma clipping return np.mean(run_times), np.std(run_times) def generate_table_row( self, model_name: str, t_onnx: tuple[float, float], t_engine: tuple[float, float], model_info: tuple[float, float, float, float], ): """Generate a table row string with model performance metrics. Args: model_name (str): Name of the model. t_onnx (tuple): ONNX model inference time statistics (mean, std). t_engine (tuple): TensorRT engine inference time statistics (mean, std). model_info (tuple): Model information (layers, params, gradients, flops). Returns: (str): Formatted table row string with model metrics. """ _layers, params, _gradients, flops = model_info return ( f"| {model_name:18s} | {self.imgsz} | - | {t_onnx[0]:.1f}±{t_onnx[1]:.1f} ms | {t_engine[0]:.1f}±" f"{t_engine[1]:.1f} ms | {params / 1e6:.1f} | {flops:.1f} |" ) @staticmethod def generate_results_dict( model_name: str, t_onnx: tuple[float, float], t_engine: tuple[float, float], model_info: tuple[float, float, float, float], ): """Generate a dictionary of profiling results. Args: model_name (str): Name of the model. t_onnx (tuple): ONNX model inference time statistics (mean, std). t_engine (tuple): TensorRT engine inference time statistics (mean, std). model_info (tuple): Model information (layers, params, gradients, flops). Returns: (dict): Dictionary containing profiling results. """ _layers, params, _gradients, flops = model_info return { "model/name": model_name, "model/parameters": params, "model/GFLOPs": round(flops, 3), "model/speed_ONNX(ms)": round(t_onnx[0], 3), "model/speed_TensorRT(ms)": round(t_engine[0], 3), } @staticmethod def print_table(table_rows: list[str]): """Print a formatted table of model profiling results. Args: table_rows (list[str]): List of formatted table row strings. """ gpu = torch.cuda.get_device_name(0) if torch.cuda.is_available() else "GPU" headers = [ "Model", "size
(pixels)", "mAPval
50-95", f"Speed
CPU ({get_cpu_info()}) ONNX
(ms)", f"Speed
{gpu} TensorRT
(ms)", "params
(M)", "FLOPs
(B)", ] header = "|" + "|".join(f" {h} " for h in headers) + "|" separator = "|" + "|".join("-" * (len(h) + 2) for h in headers) + "|" LOGGER.info(f"\n\n{header}") LOGGER.info(separator) for row in table_rows: LOGGER.info(row)