49 lines
2.0 KiB
Python
49 lines
2.0 KiB
Python
from io import BytesIO
|
|
from urllib.request import urlopen
|
|
import librosa
|
|
from transformers import Qwen2AudioForConditionalGeneration, AutoProcessor
|
|
|
|
model_name = r".\QWen\Qwen2-Audio-7B-Instruct"
|
|
processor = AutoProcessor.from_pretrained(model_name)
|
|
model = Qwen2AudioForConditionalGeneration.from_pretrained(model_name, device_map="cuda")
|
|
|
|
# conversation = [
|
|
# {"role": "user", "content": [
|
|
# {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/guess_age_gender.wav"},
|
|
# ]},
|
|
# {"role": "assistant", "content": "Yes, the speaker is female and in her twenties."},
|
|
# {"role": "user", "content": [
|
|
# {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/translate_to_chinese.wav"},
|
|
# ]},
|
|
# ]
|
|
|
|
# 定义对话
|
|
conversation = [
|
|
{"role": "user", "content": [
|
|
{"type": "audio", "audio_path": r".\QWen\Qwen2-Audio-7B-Instruct\guess_age_gender.wav"},
|
|
]},
|
|
{"role": "assistant", "content": "Yes, the speaker is female and in her twenties."},
|
|
{"role": "user", "content": [
|
|
{"type": "audio", "audio_path": r".\QWen\Qwen2-Audio-7B-Instruct\translate_to_chinese.wav"},
|
|
]},
|
|
]
|
|
|
|
text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
|
|
audios = []
|
|
for message in conversation:
|
|
if isinstance(message["content"], list):
|
|
for ele in message["content"]:
|
|
if ele["type"] == "audio":
|
|
audios.append(librosa.load(
|
|
BytesIO(urlopen(ele['audio_url']).read()),
|
|
sr=processor.feature_extractor.sampling_rate)[0]
|
|
)
|
|
|
|
inputs = processor(text=text, audios=audios, return_tensors="pt", padding=True)
|
|
inputs.input_ids = inputs.input_ids.to("cuda")
|
|
|
|
generate_ids = model.generate(**inputs, max_length=256)
|
|
generate_ids = generate_ids[:, inputs.input_ids.size(1):]
|
|
|
|
response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
|
|
print("Answer:", response) |