From f352498bbb6c7b1e0042c83dcffe9baf1908cefa Mon Sep 17 00:00:00 2001 From: Jingyu Xin Date: Thu, 11 Jun 2026 15:59:47 -0700 Subject: [PATCH] Remove examples/diffusers/eval example The diffusers image-quality eval scripts (ImageReward/CLIP-IQA/CLIP) are no longer maintained. Delete examples/diffusers/eval/ and drop the now-dangling "Evaluate Accuracy" references from examples/diffusers/README.md. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Jingyu Xin --- examples/diffusers/README.md | 57 +----------------- examples/diffusers/eval/main.py | 59 ------------------- .../diffusers/eval/metrics/imagereward.py | 36 ----------- examples/diffusers/eval/metrics/multimodal.py | 50 ---------------- examples/diffusers/eval/requirements.txt | 2 - examples/diffusers/eval/utils.py | 57 ------------------ 6 files changed, 2 insertions(+), 259 deletions(-) delete mode 100644 examples/diffusers/eval/main.py delete mode 100644 examples/diffusers/eval/metrics/imagereward.py delete mode 100644 examples/diffusers/eval/metrics/multimodal.py delete mode 100644 examples/diffusers/eval/requirements.txt delete mode 100644 examples/diffusers/eval/utils.py diff --git a/examples/diffusers/README.md b/examples/diffusers/README.md index 8fc32d7a324..c503d0492ad 100644 --- a/examples/diffusers/README.md +++ b/examples/diffusers/README.md @@ -1,6 +1,6 @@ # Diffusers Model Optimizations -Model Optimizer supports techniques like Cache Diffusion and Quantization for Diffusion models, along with scripts to evaluate models using popular evaluation metrics. +Model Optimizer supports techniques like Cache Diffusion and Quantization for Diffusion models. Post-training quantization (PTQ) is an effective model optimization technique that compresses your models to lower precision like INT8, FP8, NVFP4, etc. Quantization with Model Optimizer can compress model size by 2x-4x, speeding up inference while preserving model quality. Quantization-Aware Training (QAT) is a powerful technique for optimizing your models, particularly when PTQ methods fail to meet the requirements for your tasks. @@ -20,7 +20,6 @@ Cache Diffusion is a technique that reuses cached outputs from previous diffusio | Quantization Aware Distillation (QAD) | Example scripts on how to run QAD on diffusion models | \[[Link](#quantization-aware-distillation-qad)\] | \[[docs](https://nvidia.github.io/Model-Optimizer/guides/1_quantization.html)\] | | Build and Run with TensorRT | How to build and run your quantized model with TensorRT | \[[Link](#build-and-run-with-tensorrt-compiler-framework)\] | | | LoRA | Fuse your LoRA weights prior to quantization | \[[Link](#lora)\] | | -| Evaluate Accuracy | Evaluate your model's accuracy! | \[[Link](#evaluate-accuracy)\] | | | Pre-Quantized Checkpoints | Ready to deploy Hugging Face pre-quantized checkpoints | \[[Link](#pre-quantized-checkpoints)\] | | | Resources | Extra links to relevant resources | \[[Link](#resources)\] | | @@ -43,7 +42,7 @@ pip install nvidia-modelopt[onnx,hf] pip install -r requirements.txt ``` -Each subsection (eval, etc.) may have their own `requirements.txt` file that needs to be installed separately. +Each subsection (fastgen, distillation, etc.) may have their own `requirements.txt` file that needs to be installed separately. You can find the latest TensorRT [here](https://developer.nvidia.com/tensorrt/download). @@ -472,58 +471,6 @@ Comparing with naively reducing the generation steps, cache diffusion can achiev Stable Diffusion pipelines rely heavily on random sampling operations, which include creating Gaussian noise tensors to denoise and adding noise in the scheduling step. In the quantization recipe, we don't fix the random seed. As a result, every time you run the calibration pipeline, you could get different quantizer amax values. This may lead to the generated images being different from the ones generated with the original model. We suggest to run a few more times and choose the best one. -## Evaluate Accuracy - -This simple code demonstrates how to evaluate images generated by diffusion (or other generative) models using popular metrics such as [imagereward](https://arxiv.org/abs/2304.05977), [clip-iqa](https://arxiv.org/abs/2207.12396), and [clip](https://arxiv.org/abs/2104.08718). - -### Install Requirements - -```bash -pip install -r eval/requirments.txt -``` - -### Data Format - -Prepare a JSON file with your prompts and corresponding images in the structure below: - -```json -[ - { - "prompt": "YOUR_PROMPT", - "images": { - "MODEL_NAME": "PATH_TO_THE_IMAGE", - "MODEL_NAME": "PATH_TO_THE_IMAGE", - ... - } - }, - ... -] -``` - -- `prompt`: The text prompt used to generate the images. -- `images`: Key-value pairs of model names and image file paths. - -### Evaluate - -Run the evaluation script with your JSON file: - -```bash -python eval/main.py --data-path {PATH_TO_THE_IMAGE_JSON_PATH} --metrics imagereward -``` - -- `--data-path`: Path to your JSON file. -- `--metrics`: One or more metrics to compute (e.g. imagereward, clip-iqa, clip). - -### Sample results - -Example metrics obtained with 30 sampling steps on a set of 1K prompts (values will vary based on data and model configurations): - -| Model | Precision | ImageReward | CLIP-IQA | CLIP | -|:------------:|:------------:|:------------:|:------------:|:------------:| -| FLUX 1 Dev | BF16 | 1.118 | 0.927 | 30.15 | -| | FP4 PTQ | 1.096 | 0.923 | 29.86 | -| | FP4 QAT | 1.119 | 0.928 | 29.919 | - ## Pre-Quantized Checkpoints - Ready-to-deploy checkpoints \[[🤗 Hugging Face - Black Forest Labs](https://huggingface.co/black-forest-labs)\] diff --git a/examples/diffusers/eval/main.py b/examples/diffusers/eval/main.py deleted file mode 100644 index f4ff2d2e3bc..00000000000 --- a/examples/diffusers/eval/main.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -from pathlib import Path - -from metrics.imagereward import compute_image_reward_metrics -from metrics.multimodal import compute_clip, compute_clip_iqa -from utils import load_json_file - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--data-path", type=str, required=True, help="Path to the data file.") - parser.add_argument( - "--metrics", - nargs="+", # or nargs="*" - default=["imagereward"], - choices=["imagereward", "clip-iqa", "clip"], - help="Model IDs to run.", - ) - - return parser.parse_args() - - -def main(args): - data = load_json_file(Path(args.data_path)) - results = [] - if "imagereward" in args.metrics: - results.append(compute_image_reward_metrics(data)) - - if "clip-iqa" in args.metrics: - results.append(compute_clip_iqa(data)) - - if "clip" in args.metrics: - results.append(compute_clip(data)) - - if not results: - raise NotImplementedError( - "No recognized metrics were provided. Available: 'imagereward', 'clip-iqa', 'clip'" - ) - print(results) - - -if __name__ == "__main__": - args = parse_arguments() - main(args) diff --git a/examples/diffusers/eval/metrics/imagereward.py b/examples/diffusers/eval/metrics/imagereward.py deleted file mode 100644 index cb7fdbfec9a..00000000000 --- a/examples/diffusers/eval/metrics/imagereward.py +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import defaultdict - -import ImageReward as ImageReward -import numpy as np -from tqdm import tqdm - -# Load your model once outside the function -image_reward_model = ImageReward.load("ImageReward-v1.0", device="cuda") - - -def compute_image_reward_metrics(data): - scores = defaultdict(list) - for item in tqdm(data, desc="Computing image reward metrics"): - prompt = item["prompt"] - for model_name, image_path in item["images"].items(): - score = image_reward_model.score(prompt, image_path) - scores[model_name].append(score) - - # Compute the mean for each model - results = {model_name: np.mean(score_list) for model_name, score_list in scores.items()} - return results diff --git a/examples/diffusers/eval/metrics/multimodal.py b/examples/diffusers/eval/metrics/multimodal.py deleted file mode 100644 index 04cc94b070f..00000000000 --- a/examples/diffusers/eval/metrics/multimodal.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import defaultdict - -import torch -from torchmetrics.multimodal import CLIPImageQualityAssessment, CLIPScore -from tqdm import tqdm -from utils import convert_img2tensor, reorganize_data - - -def compute_clip_iqa(data, device: str = "cuda"): - results = defaultdict(list) - reorg_data = reorganize_data(data) - for model_name in reorg_data: - clip_model = CLIPImageQualityAssessment( - model_name_or_path="openai/clip-vit-large-patch14" - ).to(device) - for entry in tqdm(reorg_data[model_name], desc=f"Computing CLIP-IQA for {model_name}"): - img_path = entry["image"] - image_tensor = convert_img2tensor(img_path) - clip_model.update(image_tensor.to(torch.float32).to(device).unsqueeze(0)) - results[model_name] = clip_model.compute().mean().item() - return {"CLIP-IQA": results} - - -def compute_clip(data, device: str = "cuda"): - results = defaultdict(list) - reorg_data = reorganize_data(data) - for model_name in reorg_data: - clip_model = CLIPScore(model_name_or_path="openai/clip-vit-large-patch14").to(device) - for entry in tqdm(reorg_data[model_name], desc=f"Computing CLIP for {model_name}"): - prompt = entry["prompt"] - img_path = entry["image"] - image_tensor = convert_img2tensor(img_path) - clip_model.update(image_tensor.to(torch.float32).to(device).unsqueeze(0), prompt) - results[model_name] = clip_model.compute().mean().item() - return {"CLIP": results} diff --git a/examples/diffusers/eval/requirements.txt b/examples/diffusers/eval/requirements.txt deleted file mode 100644 index 4194abaf566..00000000000 --- a/examples/diffusers/eval/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -image-reward -torchmetrics diff --git a/examples/diffusers/eval/utils.py b/examples/diffusers/eval/utils.py deleted file mode 100644 index b85995f0dec..00000000000 --- a/examples/diffusers/eval/utils.py +++ /dev/null @@ -1,57 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from collections import defaultdict -from pathlib import Path - -import numpy as np -import torch -from PIL import Image - - -def load_json_file(file_path: Path): - with open(file_path) as f: - data = json.load(f) - return data - - -def convert_img2tensor(image_path): - image_data = Image.open(image_path).convert("RGB") - image_tensor = torch.from_numpy(np.array(image_data)).permute(2, 0, 1) - return image_tensor - - -def reorganize_data(data): - """ - data: a list of dicts, each dict like: - { - "prompt": , - "images": { - "modelA": , - "modelB": , - ... - } - } - returns: dict of model_name -> list of { "prompt": <>, "image": <> } - """ - model_dict = defaultdict(list) - - for item in data: - prompt = item["prompt"] - for model_name, image_path in item["images"].items(): - model_dict[model_name].append({"prompt": prompt, "image": image_path}) - - return dict(model_dict)