sudo apt install --no-install-recommends google-perftools
sudo apt install bc

$ git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui.git
$ cd stable-diffusion-webui

$ bash ./webui.sh

################################################################
Install script for stable-diffusion + Web UI
Tested on Debian 11 (Bullseye), Fedora 34+ and openSUSE Leap 15.4 or newer.
################################################################
...
Downloading: "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors" to /home/xxx/yyy/stable-diffusion-webui/models/Stable-diffusion/v1-5-pruned-emaonly.safetensors

 85%|██████████████████████████████████████████████████████████████████            | 3.36G/3.97G [00:53<00:15, 42.0MB/s]

$du -h
...
4.0G    ./models/Stable-diffusion
...
4.3G    .

$ ls -1 ~/.cache/huggingface/hub/
models--openai--clip-vit-large-patch14
models--runwayml--stable-diffusion-v1-5
models--stabilityai--stable-diffusion-xl-base-1.0
version.txt
version_diffusers_cache.txt

from diffusers import AutoPipelineForText2Image
import torch

pipeline = AutoPipelineForText2Image.from_pretrained(
    "runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True
).to("cuda")

$ pip install 'huggingface_hub[cli,torch]'
$ huggingface-cli login

model_id = "stabilityai/stable-diffusion-3-medium-diffusers"

pipeline = DiffusionPipeline.from_pretrained(
    model_id,
    torch_dtype=torch.float16,
)
pipeline.to("cuda")

model_id = "stabilityai/stable-diffusion-3-medium-diffusers"

pipeline = DiffusionPipeline.from_pretrained(
    model_id,
    text_encoder_3=None,
    tokenizer_3=None,
    torch_dtype=torch.float16,
)
pipeline.to("cuda")

pip install peft

$ cd ~/.cache/huggingface/hub/models--cagliostrolab--animagine-xl-3.1/blobs
$ ls -l

$ pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu121
$ pip install -r requirement.txt

$ python app.py

$ cd sketch2lineart
$ du -h
2.4G    ./controlnet
362M    ./tagger
82M     ./lora
24K     ./utils
...
2.8G    .

$ cd ~/.cache/huggingface/hub
$ du -h models--cagliostrolab--animagine-xl-3.1
6.4G    models--cagliostrolab--animagine-xl-3.1
$ du -h models--madebyollin--sdxl-vae-fp16-fix
320M    models--madebyollin--sdxl-vae-fp16-fix

config.json: 100%|█████████████████████████████████████████████████████████████████████| 631/631 [00:00<00:00, 2.66MB/s]
diffusion_pytorch_model.safetensors: 100%|███████████████████████████████████████████| 335M/335M [00:03<00:00, 92.7MB/s]
model_index.json: 100%|████████████████████████████████████████████████████████████████| 680/680 [00:00<00:00, 3.98MB/s]
text_encoder/config.json: 100%|████████████████████████████████████████████████████████| 560/560 [00:00<00:00, 6.09MB/s]
text_encoder_2/config.json: 100%|██████████████████████████████████████████████████████| 570/570 [00:00<00:00, 4.46MB/s]
tokenizer/tokenizer_config.json: 100%|█████████████████████████████████████████████████| 705/705 [00:00<00:00, 6.81MB/s]
scheduler/scheduler_config.json: 100%|█████████████████████████████████████████████████| 464/464 [00:00<00:00, 4.31MB/s]
tokenizer/special_tokens_map.json: 100%|███████████████████████████████████████████████| 588/588 [00:00<00:00, 5.20MB/s]
tokenizer_2/tokenizer_config.json: 100%|███████████████████████████████████████████████| 856/856 [00:00<00:00, 5.86MB/s]
tokenizer_2/special_tokens_map.json: 100%|█████████████████████████████████████████████| 462/462 [00:00<00:00, 3.76MB/s]
tokenizer/merges.txt: 100%|██████████████████████████████████████████████████████████| 525k/525k [00:00<00:00, 1.11MB/s]
unet/config.json: 100%|████████████████████████████████████████████████████████████| 1.77k/1.77k [00:00<00:00, 11.3MB/s]
tokenizer/vocab.json: 100%|████████████████████████████████████████████████████████| 1.06M/1.06M [00:00<00:00, 1.81MB/s]
model.safetensors: 100%|█████████████████████████████████████████████████████████████| 246M/246M [00:06<00:00, 35.7MB/s]
diffusion_pytorch_model.safetensors: 100%|██████████████████████████████████████████| 5.14G/5.14G [00:19<00:00, 264MB/s]
model.safetensors: 100%|███████████████████████████████████████████████████████████| 1.39G/1.39G [01:00<00:00, 22.9MB/s]
Fetching 16 files: 100%|████████████████████████████████████████████████████████████████| 16/16 [01:00<00:00,  3.81s/it]

Loading pipeline components...: 100%|█████████████████████████████████████████████████████| 7/7 [00:01<00:00,  5.70it/s]
masterpiece, best quality, monochrome, greyscale, lineart, white background
100%|███████████████████████████████████████████████████████████████████████████████████| 30/30 [00:45<00:00,  1.51s/it]
Time taken: 53.10483503341675

$ pip list | grep pytorch-lightning
pytorch-lightning         2.3.1

pip install -qU pytorch-lightning

$ curl -LO https://huggingface.co/gsdf/Counterfeit-V2.5/resolve/main/Counterfeit-V2.5.vae.pt
$ curl -LO https://raw.githubusercontent.com/huggingface/diffusers/main/scripts/convert_vae_pt_to_diffusers.py
$ python convert_vae_pt_to_diffusers.py --vae_pt_path Counterfeit-V2.5.vae.pt --dump_path counterfeit_vae

$ curl -LO https://huggingface.co/gsdf/Counterfeit-V2.5/resolve/main/vae/config.json
$ curl -LO https://huggingface.co/gsdf/Counterfeit-V2.5/resolve/main/vae/diffusion_pytorch_model.safetensors

import torch
from diffusers import (
    StableDiffusionPipeline,
    AutoencoderKL,
)

hub_dir = Path(os.getenv("HOME"))/".cache/huggingface/hub"
vae_st = str(hub_dir/"models--gsdf--Counterfeit-V2.5-vae/diffusion_pytorch_model.safetensors")
vae = AutoencoderKL.from_single_file(vae_st, torch_dtype=torch.float16).to_empty(device="cuda")

pipe = StableDiffusionPipeline.from_pretrained(
    "gsdf/Counterfeit-V2.5",
    #vae=vae,
    torch_dtype=torch.float16
).to("cuda")

pipe.load_textual_inversion(
    "gsdf/Counterfeit-V3.0",
    weight_name="embedding/EasyNegativeV2.safetensors", 
    token="EasyNegativeV2"
)

result = pipe(prompt="girl eating pizza", negative_prompt="EasyNegativeV2", width=512, height=512, num_inference_steps=200)

display(result.images[0])

$ mkdir ~/.cache/huggingface/hub/models--gsdf--Counterfeit-V3.0
$ cd ~/.cache/huggingface/hub/models--gsdf--Counterfeit-V3.0
$ curl -LO https://huggingface.co/gsdf/Counterfeit-V3.0/resolve/main/Counterfeit-V3.0_fix_fp16.safetensors

from __future__ import annotations

import os
from pathlib import Path

import torch
from diffusers import StableDiffusionPipeline


hub_dir = Path(os.getenv("HOME"))/".cache/huggingface/hub"
model = str(hub_dir/"models--gsdf--Counterfeit-V3.0/Counterfeit-V3.0_fix_fp16.safetensors")

pipe = StableDiffusionPipeline.from_single_file(
    model,
    text_encoder_3=None,
    tokenizer_3=None,
    variant="fp16",
    torch_dtype=torch.float16
).to("cuda")

pipe.load_textual_inversion(
    "gsdf/Counterfeit-V3.0",
    weight_name="embedding/EasyNegativeV2.safetensors", 
    token="EasyNegativeV2"
)

result = pipe(
    prompt="girl eating pizza",
    negative_prompt="EasyNegativeV2",
    width=512,
    height=512,
    num_inference_steps=200
)

display(result.images[0])

$ open-clip-torch
$ pip install -U "diffusers[training]"
Successfully installed absl-py-2.1.0 accelerate-0.32.1 markdown-3.6 protobuf-3.20.3 tensorboard-2.17.0 tensorboard-data-server-0.7.2 werkzeug-3.0.3

ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
protoc-gen-openapiv2 0.0.1 requires protobuf>=4.21.0, but you have protobuf 3.20.3 which is incompatible.

$ pip install -U "protobuf<5"

$ nvidia-smi

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 515.86.01    Driver Version: 515.86.01    CUDA Version: 11.7     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  NVIDIA RTX A4000    Off  | 00000000:51:00.0 Off |                  Off |
| 67%   87C    P2   139W / 140W |  10746MiB / 16376MiB |     97%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

$ mkdir ~/.cache/huggingface/hub/models--city96--FLUX.1-dev-gguf
$ cd ~/.cache/huggingface/hub/models--city96--FLUX.1-dev-gguf
$ curl -LO https://huggingface.co/city96/FLUX.1-dev-gguf/resolve/main/flux1-dev-Q8_0.gguf

hub_dir = Path(os.getenv("HOME"))/".cache/huggingface/hub"
model = str(hub_dir/"models--city96--FLUX.1-dev-gguf/flux1-dev-Q8_0.gguf")

pipeline = StableDiffusionPipeline.from_single_file(
    model,
    #text_encoder_3=None,
    #tokenizer_3=None,
    torch_dtype=torch.float16,
).to("cuda")

$ ldconfig -p | grep openblas

$ sudo apt-get install libopenblas-dev

$ sudo apt install ccache
$ git clone --recursive https://github.com/leejet/stable-diffusion.cpp
$ cd stable-diffusion.cpp
$ mkdir build
$ cd build
$ cmake .. -DGGML_OPENBLAS=ON
$ cmake --build . --config Release

$ cat /usr/local/cuda/include/cublas_api.h | grep CUBLAS_VER

$ sudo apt install ccache
$ git clone --recursive https://github.com/leejet/stable-diffusion.cpp
$ cd stable-diffusion.cpp
$ mkdir build
$ cd build
$ cmake .. -DSD_CUBLAS=ON
$ cmake --build . --config Release

$ cmake --build . --config Release
[  1%] Building C object thirdparty/CMakeFiles/zip.dir/zip.c.o
In file included from /home/xxx/git_work/stable-diffusion.cpp/thirdparty/zip.c:40:
/home/xxx/git_work/stable-diffusion.cpp/thirdparty/miniz.h:4988:9: note: ‘#pragma message: Using fopen, ftello, fseeko, stat() etc. path for file I/O - this path may not support large files.’
 4988 | #pragma message(                                                               \
      |         ^~~~~~~
[  1%] Built target zip
[  2%] Building C object ggml/src/CMakeFiles/ggml.dir/ggml.c.o
[  3%] Building C object ggml/src/CMakeFiles/ggml.dir/ggml-alloc.c.o
[  5%] Building C object ggml/src/CMakeFiles/ggml.dir/ggml-backend.c.o
[  6%] Building C object ggml/src/CMakeFiles/ggml.dir/ggml-quants.c.o
[  7%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/acc.cu.o
[  8%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/arange.cu.o
[ 10%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/argsort.cu.o
[ 11%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/binbcast.cu.o
[ 12%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/clamp.cu.o
[ 14%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/concat.cu.o
[ 15%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/conv-transpose-1d.cu.o
[ 16%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/convert.cu.o
[ 17%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/cpy.cu.o
[ 19%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/cross-entropy-loss.cu.o
[ 20%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/diagmask.cu.o
[ 21%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/dmmv.cu.o
[ 23%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/fattn-tile-f16.cu.o
[ 24%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/fattn-tile-f32.cu.o
[ 25%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/fattn.cu.o
[ 26%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/getrows.cu.o
[ 28%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/im2col.cu.o
[ 29%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/mmq.cu.o
[ 30%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/mmvq.cu.o
[ 32%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/norm.cu.o
[ 33%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/pad.cu.o
[ 34%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/pool2d.cu.o
[ 35%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/quantize.cu.o
[ 37%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/rope.cu.o
[ 38%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/scale.cu.o
[ 39%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/softmax.cu.o
[ 41%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/sumrows.cu.o
[ 42%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/tsembd.cu.o
[ 43%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/unary.cu.o
[ 44%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/upscale.cu.o
[ 46%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda.cu.o
/home/xxx/git_work/stable-diffusion.cpp/ggml/src/ggml-cuda.cu(2436): warning #177-D: function "set_ggml_graph_node_properties" was declared but never referenced
  static void set_ggml_graph_node_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
              ^

Remark: The warnings can be suppressed with "-diag-suppress <warning-number>"

/home/xxx/git_work/stable-diffusion.cpp/ggml/src/ggml-cuda.cu(2448): warning #177-D: function "ggml_graph_node_has_matching_properties" was declared but never referenced
  static bool ggml_graph_node_has_matching_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
              ^

/home/xxx/git_work/stable-diffusion.cpp/ggml/src/ggml-cuda.cu(2436): warning #177-D: function "set_ggml_graph_node_properties" was declared but never referenced
  static void set_ggml_graph_node_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
              ^

Remark: The warnings can be suppressed with "-diag-suppress <warning-number>"

/home/xxx/git_work/stable-diffusion.cpp/ggml/src/ggml-cuda.cu(2448): warning #177-D: function "ggml_graph_node_has_matching_properties" was declared but never referenced
  static bool ggml_graph_node_has_matching_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
              ^

/home/xxx/git_work/stable-diffusion.cpp/ggml/src/ggml-cuda.cu(2436): warning #177-D: function "set_ggml_graph_node_properties" was declared but never referenced
  static void set_ggml_graph_node_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
              ^

Remark: The warnings can be suppressed with "-diag-suppress <warning-number>"

/home/xxx/git_work/stable-diffusion.cpp/ggml/src/ggml-cuda.cu(2448): warning #177-D: function "ggml_graph_node_has_matching_properties" was declared but never referenced
  static bool ggml_graph_node_has_matching_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
              ^

/home/xxx/git_work/stable-diffusion.cpp/ggml/src/ggml-cuda.cu(2436): warning #177-D: function "set_ggml_graph_node_properties" was declared but never referenced
  static void set_ggml_graph_node_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
              ^

Remark: The warnings can be suppressed with "-diag-suppress <warning-number>"

/home/xxx/git_work/stable-diffusion.cpp/ggml/src/ggml-cuda.cu(2448): warning #177-D: function "ggml_graph_node_has_matching_properties" was declared but never referenced
  static bool ggml_graph_node_has_matching_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
              ^

[ 47%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu.o
[ 48%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu.o
[ 50%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu.o
[ 51%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu.o
[ 52%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu.o
[ 53%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-iq1_s.cu.o
[ 55%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-iq2_s.cu.o
[ 56%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu.o
[ 57%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu.o
[ 58%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-iq3_s.cu.o
[ 60%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu.o
[ 61%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu.o
[ 62%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu.o
[ 64%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q2_k.cu.o
[ 65%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q3_k.cu.o
[ 66%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q4_0.cu.o
[ 67%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q4_1.cu.o
[ 69%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q4_k.cu.o
[ 70%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q5_0.cu.o
[ 71%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q5_1.cu.o
[ 73%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q5_k.cu.o
[ 74%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q6_k.cu.o
[ 75%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/mmq-instance-q8_0.cu.o
[ 76%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu.o
[ 78%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu.o
[ 79%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu.o
[ 80%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu.o
[ 82%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu.o
[ 83%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu.o
[ 84%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu.o
[ 85%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu.o
[ 87%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu.o
[ 88%] Building CUDA object ggml/src/CMakeFiles/ggml.dir/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu.o
[ 89%] Building C object ggml/src/CMakeFiles/ggml.dir/ggml-aarch64.c.o
[ 91%] Linking CUDA static library libggml.a
[ 91%] Built target ggml
[ 92%] Building CXX object CMakeFiles/stable-diffusion.dir/model.cpp.o
[ 93%] Building CXX object CMakeFiles/stable-diffusion.dir/stable-diffusion.cpp.o
[ 94%] Building CXX object CMakeFiles/stable-diffusion.dir/upscaler.cpp.o
[ 96%] Building CXX object CMakeFiles/stable-diffusion.dir/util.cpp.o
[ 97%] Linking CXX static library libstable-diffusion.a
[ 97%] Built target stable-diffusion
[ 98%] Building CXX object examples/cli/CMakeFiles/sd.dir/main.cpp.o
[100%] Linking CXX executable ../../bin/sd
[100%] Built target sd

$ mkdir models
$ cd models
$ curl -LO https://huggingface.co/city96/FLUX.1-dev-gguf/resolve/main/flux1-dev-Q8_0.gguf
$ curl -LO https://huggingface.co/black-forest-labs/FLUX.1-schnell/resolve/main/ae.safetensors
$ curl -LO https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/clip_l.safetensors
$ curl -LO https://huggingface.co/comfyanonymous/flux_text_encoders/resolve/main/t5xxl_fp16.safetensors

$ du -h
22G     .

$ ./build/bin/sd --diffusion-model  models/flux1-dev-Q8_0.gguf --vae ./models/ae.safetensors --clip_l ./models/clip_l.safetensors --t5xxl ./models/t5xxl_fp16.safetensors  -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v

Option:
    n_threads:         4
    mode:              txt2img
    model_path:
    wtype:             unspecified
    clip_l_path:       ./models/clip_l.safetensors
    t5xxl_path:        ./models/t5xxl_fp16.safetensors
    diffusion_model_path:   models/flux1-dev-Q8_0.gguf
    vae_path:          ./models/ae.safetensors
    taesd_path:
    esrgan_path:
    controlnet_path:
    embeddings_path:
    stacked_id_embeddings_path:
    input_id_images_path:
    style ratio:       20.00
    normalize input image :  false
    output_path:       output.png
    init_img:
    control_image:
    clip on cpu:       false
    controlnet cpu:    false
    vae decoder on cpu:false
    strength(control): 0.90
    prompt:            a lovely cat holding a sign says 'flux.cpp'
    negative_prompt:
    min_cfg:           1.00
    cfg_scale:         1.00
    guidance:          3.50
    clip_skip:         -1
    width:             512
    height:            512
    sample_method:     euler
    schedule:          default
    sample_steps:      20
    strength(img2img): 0.75
    rng:               cuda
    seed:              42
    batch_count:       1
    vae_tiling:        false
    upscale_repeats:   1
System Info:
    BLAS = 1
    SSE3 = 1
    AVX = 1
    AVX2 = 1
    AVX512 = 1
    AVX512_VBMI = 0
    AVX512_VNNI = 1
    FMA = 1
    NEON = 0
    ARM_FMA = 0
    F16C = 1
    FP16_VA = 0
    WASM_SIMD = 0
    VSX = 0
[DEBUG] stable-diffusion.cpp:157  - Using CUDA backend
ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
ggml_cuda_init: found 1 CUDA devices:
  Device 0: NVIDIA L4, compute capability 8.9, VMM: yes
[INFO ] stable-diffusion.cpp:202  - loading clip_l from './models/clip_l.safetensors'
[INFO ] model.cpp:793  - load ./models/clip_l.safetensors using safetensors format
[DEBUG] model.cpp:861  - init from './models/clip_l.safetensors'
[INFO ] stable-diffusion.cpp:209  - loading t5xxl from './models/t5xxl_fp16.safetensors'
[INFO ] model.cpp:793  - load ./models/t5xxl_fp16.safetensors using safetensors format
[DEBUG] model.cpp:861  - init from './models/t5xxl_fp16.safetensors'
[INFO ] stable-diffusion.cpp:216  - loading diffusion model from 'models/flux1-dev-Q8_0.gguf'
[INFO ] model.cpp:790  - load models/flux1-dev-Q8_0.gguf using gguf format
[DEBUG] model.cpp:807  - init from 'models/flux1-dev-Q8_0.gguf'
[INFO ] stable-diffusion.cpp:223  - loading vae from './models/ae.safetensors'
[INFO ] model.cpp:793  - load ./models/ae.safetensors using safetensors format
[DEBUG] model.cpp:861  - init from './models/ae.safetensors'
[INFO ] stable-diffusion.cpp:235  - Version: Flux Dev
[INFO ] stable-diffusion.cpp:266  - Weight type:                 f16
[INFO ] stable-diffusion.cpp:267  - Conditioner weight type:     f16
[INFO ] stable-diffusion.cpp:268  - Diffusion model weight type: q8_0
[INFO ] stable-diffusion.cpp:269  - VAE weight type:             f32
[DEBUG] stable-diffusion.cpp:271  - ggml tensor size = 400 bytes
[INFO ] stable-diffusion.cpp:310  - set clip_on_cpu to true
[INFO ] stable-diffusion.cpp:313  - CLIP: Using CPU backend
[DEBUG] clip.hpp:171  - vocab size: 49408
[DEBUG] clip.hpp:182  -  trigger word img already in vocab
[DEBUG] ggml_extend.hpp:1046 - clip params backend buffer size =  235.06 MB(RAM) (196 tensors)
[DEBUG] ggml_extend.hpp:1046 - t5 params backend buffer size =  9083.77 MB(RAM) (219 tensors)
[DEBUG] ggml_extend.hpp:1046 - flux params backend buffer size =  12068.09 MB(VRAM) (780 tensors)
[DEBUG] ggml_extend.hpp:1046 - vae params backend buffer size =  94.57 MB(VRAM) (138 tensors)
[DEBUG] stable-diffusion.cpp:398  - loading weights
[DEBUG] model.cpp:1530 - loading tensors from ./models/clip_l.safetensors
[DEBUG] model.cpp:1530 - loading tensors from ./models/t5xxl_fp16.safetensors
[INFO ] model.cpp:1685 - unknown tensor 'text_encoders.t5xxl.encoder.embed_tokens.weight | f16 | 2 [4096, 32128, 1, 1, 1]' in model file
[DEBUG] model.cpp:1530 - loading tensors from models/flux1-dev-Q8_0.gguf
[DEBUG] model.cpp:1530 - loading tensors from ./models/ae.safetensors
[INFO ] stable-diffusion.cpp:482  - total params memory size = 21481.50MB (VRAM 12162.66MB, RAM 9318.83MB): clip 9318.83MB(RAM), unet 12068.09MB(VRAM), vae 94.57MB(VRAM), controlnet 0.00MB(VRAM), pmid 0.00MB(RAM)
[INFO ] stable-diffusion.cpp:501  - loading model from '' completed, taking 95.26s
[INFO ] stable-diffusion.cpp:518  - running in Flux FLOW mode
[DEBUG] stable-diffusion.cpp:572  - finished loaded file
[DEBUG] stable-diffusion.cpp:1378 - txt2img 512x512
[DEBUG] stable-diffusion.cpp:1127 - prompt after extract and remove lora: "a lovely cat holding a sign says 'flux.cpp'"
[INFO ] stable-diffusion.cpp:655  - Attempting to apply 0 LoRAs
[INFO ] stable-diffusion.cpp:1132 - apply_loras completed, taking 0.00s
[DEBUG] conditioner.hpp:1036 - parse 'a lovely cat holding a sign says 'flux.cpp'' to [['a lovely cat holding a sign says 'flux.cpp'', 1], ]
[DEBUG] clip.hpp:311  - token length: 77
[DEBUG] t5.hpp:397  - token length: 256
[DEBUG] ggml_extend.hpp:998  - t5 compute buffer size: 68.25 MB(RAM)
[DEBUG] conditioner.hpp:1155 - computing condition graph completed, taking 35379 ms
[INFO ] stable-diffusion.cpp:1256 - get_learned_condition completed, taking 35384 ms
[INFO ] stable-diffusion.cpp:1279 - sampling using Euler method
[INFO ] stable-diffusion.cpp:1283 - generating image: 1/1 - seed 42
[DEBUG] ggml_extend.hpp:998  - flux compute buffer size: 398.50 MB(VRAM)
  |==================================================| 20/20 - 1.10s/it
[INFO ] stable-diffusion.cpp:1315 - sampling completed, taking 42.70s
[INFO ] stable-diffusion.cpp:1323 - generating 1 latent images completed, taking 43.16s
[INFO ] stable-diffusion.cpp:1326 - decoding 1 latents
[DEBUG] ggml_extend.hpp:998  - vae compute buffer size: 1664.00 MB(VRAM)
[DEBUG] stable-diffusion.cpp:987  - computing vae [mode: DECODE] graph completed, taking 1.10s
[INFO ] stable-diffusion.cpp:1336 - latent 1 decoded, taking 1.10s
[INFO ] stable-diffusion.cpp:1340 - decode_first_stage completed, taking 1.10s
[INFO ] stable-diffusion.cpp:1449 - txt2img completed in 79.65s
save result image to 'output.png'

$ nvidia-smi
Mon Sep  9 23:33:04 2024
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.90.07              Driver Version: 550.90.07      CUDA Version: 12.4     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA L4                      On  |   00000000:00:04.0 Off |                    0 |
| N/A   49C    P0             30W /   72W |   12359MiB /  23034MiB |      0%      Default |
|                                         |                        |                  N/A |
+-----------------------------------------+------------------------+----------------------+

./build/bin/sd --diffusion-model  models/flux1-dev-Q8_0.gguf --vae ./models/ae.safetensors --clip_l ./models/clip_l.safetensors --t5xxl ./models/t5xxl_fp16.safetensors  -p "A muscular macho male warrior is holding a large sword. The sword is covered with flames." --cfg-scale 1.0 --sampling-method euler -v

A muscular macho male warrior wearing golden armor is holding a large sword. The sword is covered with flames. He is now fighting with a dragon.

Anime image. Comic image. A muscular macho male warrior wearing golden armor is holding a large sword. The sword is covered with flames. He is now fighting with a dragon.

An anime-style muscular macho male warrior wearing golden armor is holding a large sword. The sword is covered with flames. He is now fighting against a dragon.

A winged goddess flies in the sky and watches over the earth.

$ pip install bitsandbytes
Successfully installed bitsandbytes-0.43.3

$ pip install -U git+https://github.com/huggingface/diffusers@c795c82df39620e2576ccda765b6e67e849c36e7
Successfully installed diffusers-0.31.0.dev0

from diffusers import BitsAndBytesConfig

$ pip install sentencepiece

$ df -T
Filesystem     Type    1K-blocks     Used Available Use% Mounted on
shm            tmpfs       65536        0     65536   0% /dev/shm
tmpfs          tmpfs    32783196        0  32783196   0% /sys/fs/cgroup
/dev/sda2      ext4    479079112 49726888 404942832  11% /dev/termination-log
overlay        overlay 103019448 47990824  55012240  47% /

%%time

nf4_id = "sayakpaul/flux.1-dev-nf4-with-bnb-integration"
model_nf4 = FluxTransformer2DModel.from_pretrained(nf4_id, torch_dtype=torch.bfloat16)
print(model_nf4.dtype)
print(model_nf4.config.quantization_config)

$ df -T
...
overlay        overlay 103019448 54533208  48469856  53% /

from huggingface_hub import login
login(token="your_access_token")

%%time

model_id = "black-forest-labs/FLUX.1-dev"
pipe = FluxPipeline.from_pretrained(model_id, transformer=model_nf4, torch_dtype=torch.bfloat16)
pipe.enable_model_cpu_offload()

$ df -T
...
overlay        overlay 103019448 64243628  38759436  63% /

$ cd ~/.cache/huggingface/hub/models--black-forest-labs--FLUX.1-dev
$ du -h
4.0K    ./snapshots/0ef5fff789c832c5c7f4e127f94c8b54bbcced44/tokenizer
8.0K    ./snapshots/0ef5fff789c832c5c7f4e127f94c8b54bbcced44/vae
4.0K    ./snapshots/0ef5fff789c832c5c7f4e127f94c8b54bbcced44/scheduler
12K     ./snapshots/0ef5fff789c832c5c7f4e127f94c8b54bbcced44/text_encoder_2
8.0K    ./snapshots/0ef5fff789c832c5c7f4e127f94c8b54bbcced44/text_encoder
8.0K    ./snapshots/0ef5fff789c832c5c7f4e127f94c8b54bbcced44/tokenizer_2
48K     ./snapshots/0ef5fff789c832c5c7f4e127f94c8b54bbcced44
52K     ./snapshots
8.0K    ./refs
9.3G    ./blobs
9.3G    .

prompt = "A mystic cat with a sign that says hello world!"
image = pipe(prompt, guidance_scale=3.5, num_inference_steps=50, generator=torch.manual_seed(0)).images[0]
image.save("flux-nf4-dev-loaded.png")

$ cd ~/.cache/huggingface/hub
$ rm -rf models--sayakpaul--flux.1-dev-nf4-with-bnb-integration

from transformers import T5EncoderModel
from diffusers import FluxTransformer2DModel, FluxPipeline
import torch
import gc


def flush():
    """Wipes off memory."""
    gc.collect()
    torch.cuda.empty_cache()
    torch.cuda.reset_max_memory_allocated()
    torch.cuda.reset_peak_memory_stats()


def bytes_to_giga_bytes(bytes):
    return f"{(bytes / 1024 / 1024 / 1024):.3f}"

flush()

%%time

nf4_model_id = "sayakpaul/flux.1-dev-nf4-pkg"
text_encoder_2 = T5EncoderModel.from_pretrained(
    nf4_model_id, subfolder="text_encoder_2", torch_dtype=torch.float16
)
transformer = FluxTransformer2DModel.from_pretrained(
    nf4_model_id, subfolder="transformer", torch_dtype=torch.float16
)

$ cd ~/.cache/huggingface/hub/models--sayakpaul--flux.1-dev-nf4-pkg
$ $ du -h
8.0K    ./snapshots/da15611221570dbf0ad4f620165f0072e443e3c4/transformer
8.0K    ./snapshots/da15611221570dbf0ad4f620165f0072e443e3c4/text_encoder_2
20K     ./snapshots/da15611221570dbf0ad4f620165f0072e443e3c4
24K     ./snapshots
8.0K    ./refs
4.0K    ./.no_exist/da15611221570dbf0ad4f620165f0072e443e3c4/transformer
8.0K    ./.no_exist/da15611221570dbf0ad4f620165f0072e443e3c4
12K     ./.no_exist
13G     ./blobs
13G     .

%%time

pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-dev",
    text_encoder_2=text_encoder_2,
    transformer=transformer,
    torch_dtype=torch.float16
)
pipe.enable_model_cpu_offload()

%%time

prompt = "A mystic cat with a sign that says hello world!"
image = pipe(
    prompt,
    guidance_scale=3.5,
    num_inference_steps=50,
    generator=torch.manual_seed(0)
).images[0]

torch.cuda.empty_cache()
memory = bytes_to_giga_bytes(torch.cuda.memory_allocated())
print(f"{memory=} GB.")

image.save("flux-nf4-dev.png")

$ nvidia-smi
...
| 41%   61C    P2   138W / 140W |  15098MiB / 16376MiB |    100%      Default |
...

$ export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
$ jupyter lab

$ nvidia-smi
...
| 59%   80C    P2   139W / 140W |  14566MiB / 16376MiB |    100%      Default |

$ df -T
...
overlay        overlay 103019448 70426584  32576480  69% /