「Diffusers」を試す 5. 量子化

!pip install -U bitsandbytes

import torch
from diffusers import DiffusionPipeline
from diffusers.quantizers import PipelineQuantizationConfig

# 量子化の設定
pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend="bitsandbytes_4bit",  # bitsandbytesを使って4ビット量子化
    quant_kwargs={
        "load_in_4bit": True,
        "bnb_4bit_quant_type": "nf4",
        "bnb_4bit_compute_dtype": torch.bfloat16
    },
    components_to_quantize=["transformer", "text_encoder_2"],
)

# 量子化設定を from_pretrained()` に渡してパイプラインを量子化。
pipe = DiffusionPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    quantization_config=pipeline_quant_config,
    torch_dtype=torch.bfloat16,
).to("cuda")

pipe("photo of a cute dog").images[0]

OutOfMemoryError: CUDA out of memory. Tried to allocate 18.00 MiB. GPU 0 has a total capacity of 22.16 GiB of which 13.38 MiB is free. Process 29705 has 22.14 GiB memory in use. Of the allocated memory 21.96 GiB is allocated by PyTorch, and 6.29 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

!pip install optimum-quanto

import torch
from diffusers import DiffusionPipeline
from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig
from diffusers.quantizers.quantization_config import QuantoConfig
from diffusers.quantizers import PipelineQuantizationConfig
from transformers import BitsAndBytesConfig as TransformersBitsAndBytesConfig

# quant_mapping を使った高度な量子化の設定
pipeline_quant_config = PipelineQuantizationConfig(
    quant_mapping={
        # transformer に QuantoConfig を使った量子化を設定
        "transformer": QuantoConfig(weights_dtype="int8"),
        # テキストエンコーダに transformers.BitsAndBytesConfig を使った量子化を設定
        "text_encoder_2": TransformersBitsAndBytesConfig(
            load_in_4bit=True, compute_dtype=torch.bfloat16
        ),
    }
)

pipe = DiffusionPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    quantization_config=pipeline_quant_config,
    torch_dtype=torch.bfloat16,
).to("cuda")

pipe("photo of a cute dog").images[0]

import torch
from diffusers import DiffusionPipeline
from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig
from diffusers.quantizers import PipelineQuantizationConfig
from transformers import BitsAndBytesConfig as TransformersBitsAndBytesConfig

pipeline_quant_config = PipelineQuantizationConfig(
    quant_mapping={
        # Diffusers 側のコンポーネント → diffusers.BitsAndBytesConfig
        "transformer": DiffusersBitsAndBytesConfig(
            load_in_4bit=True,
            bnb_4bit_compute_dtype=torch.bfloat16
        ),
        # Transformers 側のコンポーネント → transformers.BitsAndBytesConfig
        "text_encoder_2": TransformersBitsAndBytesConfig(
            load_in_4bit=True,
            compute_dtype=torch.bfloat16
        ),
    }
)

pipe = DiffusionPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    quantization_config=pipeline_quant_config,
    torch_dtype=torch.bfloat16,
).to("cuda")

pipe("photo of a cute dog").images[0]

!pip install diffusers transformers accelerate bitsandbytes -U

Successfully installed bitsandbytes-0.47.0 tokenizers-0.22.0 transformers-4.56.0

from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig
from transformers import BitsAndBytesConfig as TransformersBitsAndBytesConfig
import torch
from diffusers import AutoModel
from transformers import T5EncoderModel
from diffusers import FluxPipeline

quant_config = TransformersBitsAndBytesConfig(load_in_8bit=True,)

text_encoder_2_8bit = T5EncoderModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="text_encoder_2",
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(load_in_8bit=True,)

transformer_8bit = AutoModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="transformer",
    quantization_config=quant_config,
    # デフォルトだと torch.nn.LayerNorm など他のモジュールは全てtorch.float16に変換される
    # これらのデータ型を変更する場合はここ
    torch_dtype=torch.float16,
)

pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    transformer=transformer_8bit,
    text_encoder_2=text_encoder_2_8bit,
    torch_dtype=torch.float16,
    # autoにすると以下のエラーになったので、balanced にした:
    # NotImplementedError: auto not supported. Supported strategies are: balanced, cuda
    device_map="balanced",
)

pipe_kwargs = {
    "prompt": "A cat holding a sign that says hello world",
    "height": 1024,
    "width": 1024,
    "guidance_scale": 3.5,
    "num_inference_steps": 50,
    "max_sequence_length": 512,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(0),).images[0]
display(image)

`torch_dtype` is deprecated! Use `dtype` instead!

+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.54.15              Driver Version: 550.54.15      CUDA Version: 12.4     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA L4                      Off |   00000000:00:03.0 Off |                    0 |
| N/A   52C    P0             28W /   72W |   22189MiB /  23034MiB |      0%      Default |
|                                         |                        |                  N/A |
+-----------------------------------------+------------------------+----------------------+

print(transformer_8bit.get_memory_footprint())

11904493696

print(text_encoder_2_8bit.get_memory_footprint())

7914008576

from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig
from transformers import BitsAndBytesConfig as TransformersBitsAndBytesConfig
import torch
from diffusers import AutoModel
from transformers import T5EncoderModel
from diffusers import FluxPipeline

quant_config = TransformersBitsAndBytesConfig(load_in_4bit=True,)  # 4ビット量子化

text_encoder_2_8bit = T5EncoderModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="text_encoder_2",
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(load_in_4bit=True,)  # 4ビット量子化

transformer_8bit = AutoModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="transformer",
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    transformer=transformer_8bit,
    text_encoder_2=text_encoder_2_8bit,
    torch_dtype=torch.float16,
    device_map="balanced",
)

pipe_kwargs = {
    "prompt": "A cat holding a sign that says hello world",
    "height": 1024,
    "width": 1024,
    "guidance_scale": 3.5,
    "num_inference_steps": 50,
    "max_sequence_length": 512,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(0),).images[0]
display(image)

Sun Aug 31 17:20:19 2025       
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.54.15              Driver Version: 550.54.15      CUDA Version: 12.4     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA L4                      Off |   00000000:00:03.0 Off |                    0 |
| N/A   68C    P0             32W /   72W |   13227MiB /  23034MiB |      0%      Default |
|                                         |                        |                  N/A |
+-----------------------------------------+------------------------+----------------------+

from diffusers import AutoModel, FluxPipeline
from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig
from transformers import BitsAndBytesConfig as TransformersBitsAndBytesConfig
import torch
from transformers import T5EncoderModel

text_encoder_2_8bit = T5EncoderModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="text_encoder_2",
    quantization_config=TransformersBitsAndBytesConfig(
        load_in_8bit=True,      # 8ビット量子化
        llm_int8_threshold=10.0,  # 外れ値のしきい値をfloatで設定
    ),
    torch_dtype=torch.float16,
)

transformer_8bit = AutoModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="transformer",
    quantization_config=DiffusersBitsAndBytesConfig(
        load_in_8bit=True,        # 8ビット量子化
        llm_int8_threshold=10.0,  # 外れ値のしきい値をfloatで設定
    ),
    torch_dtype=torch.float16,
)

pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    transformer=transformer_8bit,
    text_encoder_2=text_encoder_2_8bit,
    torch_dtype=torch.float16,
    device_map="balanced",
)

pipe_kwargs = {
    "prompt": "A cat holding a sign that says hello world",
    "height": 1024,
    "width": 1024,
    "guidance_scale": 3.5,
    "num_inference_steps": 50,
    "max_sequence_length": 512,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(0),).images[0]
display(image)

from diffusers import DiffusionPipeline, SD3Transformer2DModel, BitsAndBytesConfig
import torch

quantization_config = BitsAndBytesConfig(
    load_in_8bit=True,                   # 8ビット量子化
    llm_int8_skip_modules=["proj_out"],  # proj_out は量子化をスキップ
)

transformer_8bit = SD3Transformer2DModel.from_pretrained(
    "stabilityai/stable-diffusion-3-medium-diffusers",
    subfolder="transformer",
    quantization_config=quantization_config,
    torch_dtype=torch.float16,
)

pipeline = DiffusionPipeline.from_pretrained(
    "stabilityai/stable-diffusion-3-medium-diffusers",
    transformer=transformer_8bit,
    torch_dtype=torch.float16,
    device_map="cuda"
)

prompt = "A cat holding a sign that says hello world"
pipeline(prompt, generator=torch.manual_seed(0)).images[0]

from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig
from transformers import BitsAndBytesConfig as TransformersBitsAndBytesConfig
from diffusers import AutoModel, FluxPipeline
from transformers import T5EncoderModel
import torch

quant_config = TransformersBitsAndBytesConfig(
    load_in_4bit=True,                      # 4ビット量子化
    bnb_4bit_compute_dtype=torch.bfloat16,  # 計算時のデータ型を bf16に
    bnb_4bit_use_double_quant=True,         # ネスト量子化を有効
)

text_encoder_2_4bit = T5EncoderModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="text_encoder_2",
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

quant_config = DiffusersBitsAndBytesConfig(
    load_in_4bit=True,                      # 4ビット量子化
    bnb_4bit_compute_dtype=torch.bfloat16,  # 計算時のデータ型を bf16に
    bnb_4bit_use_double_quant=True,         # ネスト量子化を有効
)

transformer_4bit = AutoModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="transformer",
    quantization_config=quant_config,
    torch_dtype=torch.float16,
)

pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    transformer=transformer_4bit,
    text_encoder_2=text_encoder_2_4bit,
    device_map="balanced",
)

pipe_kwargs = {
    "prompt": "A cat holding a sign that says hello world",
    "height": 1024,
    "width": 1024,
    "guidance_scale": 3.5,
    "num_inference_steps": 50,
    "max_sequence_length": 512,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(0),).images[0]
display(image)

from diffusers import BitsAndBytesConfig as DiffusersBitsAndBytesConfig
from transformers import BitsAndBytesConfig as TransformersBitsAndBytesConfig
from diffusers import AutoModel, FluxPipeline
from transformers import T5EncoderModel
import torch

torch._dynamo.config.capture_dynamic_output_shape_ops = True

text_encoder_2_8bit = T5EncoderModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="text_encoder_2",
    quantization_config=TransformersBitsAndBytesConfig(load_in_8bit=True),
    torch_dtype=torch.bfloat16,
    device_map="cuda",
)

transformer_8bit = AutoModel.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    subfolder="transformer",
    quantization_config=DiffusersBitsAndBytesConfig(load_in_8bit=True),
    torch_dtype=torch.bfloat16,
    device_map="cuda",
)
#transformer_8bit.compile(fullgraph=True)

pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    transformer=transformer_8bit,
    text_encoder_2=text_encoder_2_8bit,
    torch_dtype=torch.bfloat16,
    device_map="cuda",
)


pipe_kwargs = {
    "prompt": "A cat holding a sign that says hello world",
    "height": 1024,
    "width": 1024,
    "guidance_scale": 3.5,
    "num_inference_steps": 50,
    "max_sequence_length": 512,
}

image = pipe(**pipe_kwargs, generator=torch.manual_seed(0)).images[0]
display(image)

!pip install gguf -U

Successfully installed gguf-0.17.1

import torch
from diffusers import FluxPipeline, FluxTransformer2DModel, GGUFQuantizationConfig

ckpt_path = (
    "https://huggingface.co/city96/FLUX.1-dev-gguf/blob/main/flux1-dev-Q2_K.gguf"
)

transformer = FluxTransformer2DModel.from_single_file(
    ckpt_path,
    quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
    torch_dtype=torch.bfloat16,
)

pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    transformer=transformer,
    torch_dtype=torch.bfloat16,
)
pipe.enable_model_cpu_offload()

prompt = "A cat holding a sign that says hello world"
image = pipe(prompt, generator=torch.manual_seed(0)).images[0]
image.save("flux-gguf.png")

display(image)

!pip install -U kernels

import os

os.environ["DIFFUSERS_GGUF_CUDA_KERNELS"] = "true"

import torch
from diffusers import FluxPipeline, FluxTransformer2DModel, GGUFQuantizationConfig

ckpt_path = (
    "https://huggingface.co/city96/FLUX.1-dev-gguf/blob/main/flux1-dev-Q2_K.gguf"
)

transformer = FluxTransformer2DModel.from_single_file(
    ckpt_path,
    quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
    torch_dtype=torch.bfloat16,
)

pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    transformer=transformer,
    torch_dtype=torch.bfloat16,
)
pipe.enable_model_cpu_offload()

prompt = "A cat holding a sign that says hello world"
image = pipe(prompt, generator=torch.manual_seed(0)).images[0]
image.save("flux-gguf.png")

RuntimeError: Failed to import diffusers.pipelines.flux.pipeline_flux because of the following error (look up to see its traceback):
Failed to import diffusers.loaders.ip_adapter because of the following error (look up to see its traceback):
Kernel `Isotr0py/ggml` at revision main does not have build: torch28-cxx11-cu126-x86_64-linux

!python -c 'import torch; print(torch.__version__) '

2.8.0+cu126

pip install torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cu126

git clone https://github.com/ngxson/diffusion-to-gguf && cd diffusion-to-gguf

uv init -p 3.12
uv sync

uv pip install torch --torch-backend=auto

 + filelock==3.19.1
 + fsspec==2025.7.0
 + jinja2==3.1.6
 + markupsafe==3.0.2
 + mpmath==1.3.0
 + networkx==3.5
 + nvidia-cublas-cu12==12.8.4.1
 + nvidia-cuda-cupti-cu12==12.8.90
 + nvidia-cuda-nvrtc-cu12==12.8.93
 + nvidia-cuda-runtime-cu12==12.8.90
 + nvidia-cudnn-cu12==9.10.2.21
 + nvidia-cufft-cu12==11.3.3.83
 + nvidia-cufile-cu12==1.13.1.3
 + nvidia-curand-cu12==10.3.9.90
 + nvidia-cusolver-cu12==11.7.3.90
 + nvidia-cusparse-cu12==12.5.8.93
 + nvidia-cusparselt-cu12==0.7.1
 + nvidia-nccl-cu12==2.27.3
 + nvidia-nvjitlink-cu12==12.8.93
 + nvidia-nvtx-cu12==12.8.90
 + setuptools==80.9.0
 + sympy==1.14.0
 + torch==2.8.0+cu128
 + triton==3.4.0
 + typing-extensions==4.15.0

uv run python -c "import torch; print(torch.cuda.is_available())"

True

uv pip show torch

Name: torch
Version: 2.8.0+cu128
Location: /SOMEWHERE/diffusion-to-gguf/.venv/lib/python3.12/site-packages
Requires: filelock, fsspec, jinja2, networkx, nvidia-cublas-cu12, nvidia-cuda-cupti-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-runtime-cu12, nvidia-cudnn-cu12, nvidia-cufft-cu12, nvidia-cufile-cu12, nvidia-curand-cu12, nvidia-cusolver-cu12, nvidia-cusparse-cu12, nvidia-cusparselt-cu12, nvidia-nccl-cu12, nvidia-nvjitlink-cu12, nvidia-nvtx-cu12, setuptools, sympy, triton, typing-extensions
Required-by:

[project]
name = "diffusion-to-gguf"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
    "torch==2.8.0",
]

[tool.uv.sources]
torch = [
    { index = "torch-cu128" },
]

[[tool.uv.index]]
name = "torch-cu128"
url = "https://download.pytorch.org/whl/cu128"
explicit = true

uv sync

uv add gguf safetensors packaging

 + gguf==0.17.1
 + numpy==2.3.2
 + packaging==25.0
 + pyyaml==6.0.2
 + safetensors==0.6.2
 + tqdm==4.67.1

uv run convert_diffusion_to_gguf.py -h

usage: convert_diffusion_to_gguf.py [-h] [--outfile OUTFILE]
                                    [--outtype {F16,BF16,Q8_0,Q6_K,Q5_K_S,Q5_1,Q5_0,Q4_K_S,Q4_1,Q4_0,Q3_K_S}]
                                    [--arch {flux,sd3,ltxv,hyvid,wan,hidream,qwen}] [--bigendian]
                                    [--cache_dir CACHE_DIR] [--subfolder SUBFOLDER] [--verbose]
                                    [model]

Convert a flux model to GGUF

positional arguments:
  model                 directory containing safetensors model file

options:
  -h, --help            show this help message and exit
  --outfile OUTFILE     path to write to; default: 'model-{ftype}.gguf' ; note: {ftype} will be replaced by the
                        outtype
  --outtype {F16,BF16,Q8_0,Q6_K,Q5_K_S,Q5_1,Q5_0,Q4_K_S,Q4_1,Q4_0,Q3_K_S}
                        output quantization scheme
  --arch {flux,sd3,ltxv,hyvid,wan,hidream,qwen}
                        output model architecture
  --bigendian           model is executed on big endian machine
  --cache_dir CACHE_DIR
                        Directory to store the intermediate files when needed.
  --subfolder SUBFOLDER
                        Subfolder on the HF Hub to load checkpoints from.
  --verbose             increase output verbosity

git lfs install
git clone https://huggingface.co/Qwen/Qwen-Image

mkdir cache

uv run convert_diffusion_to_gguf.py ./Qwen-Image/transformer \
    --arch qwen \
    --cache_dir ./cache \
    --outtype Q4_K_S \
    --outfile qwen-image-q4_k_s.gguf

INFO:root:Supplied a directory.
INFO:root:Serialized merged state dict to cache/merged_state_dict.safetensors
INFO:__main__:Converting model in cache/merged_state_dict.safetensors to qwen-image-q4_k_s.gguf with quantization Q4_K_S
INFO:gguf.gguf_writer:gguf: This GGUF file is for Little Endian only
INFO:__main__:img_in.bias,                     torch.bfloat16 --> F32, shape = {3072}
WARNING:__main__:Can't quantize tensor with shape (3072, 64) to Q4_K, falling back to F16
(snip)
INFO:__main__:txt_in.bias,                     torch.bfloat16 --> F32, shape = {3072}
INFO:__main__:txt_in.weight,                   torch.bfloat16 --> Q4_K, shape = {3584, 3072}
INFO:__main__:txt_norm.weight,                 torch.bfloat16 --> F32, shape = {3584}
INFO:gguf.gguf_writer:Writing the following files:
INFO:gguf.gguf_writer:qwen-image-q4_k_s.gguf: n_tensors = 1933, total_size = 11.5G
Writing:   1%|▉                                                                | 155M/11.5G [01:10<1:14:19, 2.55Mbyte/s]