!pip uninstall -y transformers
!pip install git+https://github.com/huggingface/transformers@v4.51.3-Qwen2.5-Omni-preview
!pip install accelerate

pip install qwen-omni-utils[decord] -U

!pip install flash-attn --no-build-isolation

import soundfile as sf

from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor
from qwen_omni_utils import process_mm_info

# モデルとプロセサをロード
model_name = "Qwen/Qwen2.5-Omni-3B"
model = Qwen2_5OmniForConditionalGeneration.from_pretrained(
    model_name,
    torch_dtype="auto",
    device_map="auto",
    attn_implementation="flash_attention_2",
)
processor = Qwen2_5OmniProcessor.from_pretrained(model_name)

Thu May  1 00:17:30 2025       
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.54.15              Driver Version: 550.54.15      CUDA Version: 12.4     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA A100-SXM4-40GB          Off |   00000000:00:04.0 Off |                    0 |
| N/A   31C    P0             46W /  400W |   11949MiB /  40960MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+

conversation = [
    {
        "role": "system",
        "content": [
            {"type": "text", "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."}
        ],
    },
    {
        "role": "user",
        "content": [
            {"type": "video", "video": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4"},
        ],
    },
]

# 動画内のオーディオを使用する
USE_AUDIO_IN_VIDEO = True

# 推論準備
text = processor.apply_chat_template(
    conversation,
    add_generation_prompt=True,
    tokenize=False
)
audios, images, videos = process_mm_info(
    conversation,
    use_audio_in_video=USE_AUDIO_IN_VIDEO
)
inputs = processor(
    text=text,
    audio=audios,
    images=images,
    videos=videos,
    return_tensors="pt",
    padding=True,
    use_audio_in_video=USE_AUDIO_IN_VIDEO
)
inputs = inputs.to(model.device).to(model.dtype)

# 推論: テキストとオーディオで出力を生成
text_ids, audio = model.generate(**inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO)

text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
print(text)
sf.write(
    "output.wav",
    audio.reshape(-1).detach().cpu().numpy(),
    samplerate=24000,
)

["system\nYou are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.\nuser\n\nassistant\nOh, that's a cool drawing! It looks like you're drawing a guitar. You're using a stylus, right? Have you been practicing a lot? If you want, you can show me more of your drawings."]

WARNING:root:System prompt modified, audio output may not work as expected. Audio output mode only works when using default system prompt 'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'

Thu May  1 00:24:12 2025       
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.54.15              Driver Version: 550.54.15      CUDA Version: 12.4     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA A100-SXM4-40GB          Off |   00000000:00:04.0 Off |                    0 |
| N/A   31C    P0             46W /  400W |   20625MiB /  40960MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+

conversation = [
    {
        "role": "system",
        "content": [
            {"type": "text", "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."}
        ],
    },
    {
        "role": "user",
        "content": [
            {"type": "audio", "audio": "sample_query.wav"},
        ]
    }
]

USE_AUDIO_IN_VIDEO = True

text = processor.apply_chat_template(
    conversation,
    add_generation_prompt=True,
    tokenize=False
)
audios, images, videos = process_mm_info(
    conversation,
    use_audio_in_video=USE_AUDIO_IN_VIDEO
)
inputs = processor(
    text=text,
    audio=audios,
    images=images,
    videos=videos,
    return_tensors="pt",
    padding=True,
    use_audio_in_video=USE_AUDIO_IN_VIDEO
)
inputs = inputs.to(model.device).to(model.dtype)

text_ids, audio = model.generate(**inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO)

text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
print(text)
sf.write(
    "output.wav",
    audio.reshape(-1).detach().cpu().numpy(),
    samplerate=24000,
)

['system\nYou are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.\nuser\n\nassistant\n嗯…首先呢，比赛的紧张刺激，就像在看一场生死搏斗。然后呢，选手们超高的竞技水平，每一次比赛都是一场技术的较量。还有啊，观众的欢呼声，那种氛围特别棒。再有就是，比赛结果的不确定性，让人很期待。最后，比赛能展现人类的毅力和勇气。你要是还有啥想法或者想补充的，随时跟我说哈。']

conversation = [
    {
        "role": "system",
        "content": [
            {"type": "text", "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."},
            {"type": "text", "text": "回答は日本語で行うこと。"},
        ],
    },
    {
        "role": "user",
        "content": [
            {"type": "audio", "audio": "sample_query.wav"},
        ]
    }
]

(snip)

['system\nYou are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.回答は日本語で行うこと。\nuser\n\nassistant\n競馬の魅力は以下の5つです。まず、予想の楽しみ性があります。レースの結果を予想して楽しむことが大きな魅力ですね。次に、競争の激しさ。多くの競馬場で多くの競馬が行われ、その中から勝ち馬を選ぶのはとても刺激的です。また、レースの速さ。馬がどれだけ速く走れるかが競馬の醍醐味の一つです。そして、レースの結果が変わる瞬間の興奮。勝ち馬が決まる瞬間は、観客全員が一斉に歓声を上げます。さらに、競馬の歴史と文化。日本には長い歴史があり、その文化は世界でも知られています。この5つが競馬の魅力の主な部分です。你要是还有关于竞马的问题，欢迎随时问我哦。']