语音大模型实战：Whisper、语音克隆与端到端语音理解

语音大模型实战：Whisper、语音克隆与端到端语音理解

1. 引言

语音是人机交互最自然的方式。2024年以来，语音大模型（如 GPT-4o、Qwen2-Audio）实现了端到端的语音理解和生成。本文将覆盖语音识别（ASR）、语音合成（TTS）和语音克隆三大核心任务。

技术栈全景：

任务	模型	特点
语音识别	Whisper large-v3	多语言、高精度
语音合成	CosyVoice / Bark	自然、可控
语音克隆	GPT-SoVITS / XTTS	少样本克隆
端到端理解	Qwen2-Audio	语音→文本→推理

2. Whisper 语音识别

2.1 基础使用

import whisper

# 加载模型
model = whisper.load_model("large-v3")

# 转录
result = model.transcribe("audio.mp3", language="zh")
print(result["text"])

# 带时间戳的转录
for segment in result["segments"]:
    start = segment["start"]
    end = segment["end"]
    text = segment["text"]
    print(f"[{start:.1f}s - {end:.1f}s] {text}")

2.2 长音频处理

import whisper
import numpy as np
from pydub import AudioSegment

def transcribe_long_audio(audio_path, model_name="large-v3", chunk_minutes=10):
    """长音频分段转录"""
    model = whisper.load_model(model_name)

    # 加载音频
    audio = AudioSegment.from_file(audio_path)
    chunk_ms = chunk_minutes * 60 * 1000
    total_ms = len(audio)

    all_segments = []
    offset = 0

    for start_ms in range(0, total_ms, chunk_ms):
        end_ms = min(start_ms + chunk_ms, total_ms)
        chunk = audio[start_ms:end_ms]

        # 导出临时文件
        chunk.export("/tmp/chunk.wav", format="wav")

        # 转录
        result = model.transcribe("/tmp/chunk.wav", language="zh")

        # 调整时间戳
        for seg in result["segments"]:
            seg["start"] += start_ms / 1000
            seg["end"] += start_ms / 1000
            all_segments.append(seg)

        print(f"已处理 {end_ms/1000:.0f}s / {total_ms/1000:.0f}s")

    return all_segments

2.3 Whisper 微调

from transformers import WhisperForConditionalGeneration, WhisperProcessor
from datasets import load_dataset

model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v3")
processor = WhisperProcessor.from_pretrained("openai/whisper-large-v3")

# 强制使用中文
model.config.forced_decoder_ids = processor.get_decoder_prompt_ids(
    language="zh", task="transcribe"
)

# 微调数据集格式
# {"audio": "path/to/audio.wav", "text": "对应的文本"}

def prepare_dataset(batch):
    audio = batch["audio"]
    input = processor(
        audio["array"],
        sampling_rate=audio["sampling_rate"],
        return_tensors="pt",
    )
    batch["input_features"] = input.input_features[0]

    labels = processor.tokenizer(batch["text"]).input_ids
    batch["labels"] = labels
    return batch

3. 语音合成（TTS）

3.1 CosyVoice 语音合成

from cosyvoice.cosyvoice import CosyVoice
import torchaudio

# 加载模型
cosyvoice = CosyVoice("CosyVoice-300M")

# 基础 TTS
tts_output = cosyvoice.inference_sft(
    "你好，欢迎使用语音合成系统。",
    speaker="中文女",
)

# 保存音频
torchaudio.save("output.wav", tts_output["tts_speech"], 22050)

3.2 带情感的语音合成

# 情感控制
emotions = {
    "happy": "太好了！我非常高兴听到这个消息！",
    "sad": "很遗憾，这个结果让人失望。",
    "angry": "这完全不可接受！",
    "surprised": "什么？这怎么可能！",
}

for emotion, text in emotions.items():
    output = cosyvoice.inference_sft(text, speaker="中文女")
    torchaudio.save(f"output_{emotion}.wav", output["tts_speech"], 22050)

4. 语音克隆

4.1 GPT-SoVITS 少样本克隆

from GPT_SoVITS.inference import get_tts_wav

# 参考音频（3-10秒的目标说话人录音）
reference_audio = "speaker_sample.wav"
reference_text = "这是参考音频对应的文字内容。"

# 克隆语音
output = get_tts_wav(
    ref_wav_path=reference_audio,
    prompt_text=reference_text,
    text="要用克隆声音说的内容。",
    text_language="zh",
)

4.2 XTTS 零样本克隆

from TTS.api import TTS

# 加载 XTTS 模型
tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2")

# 零样本克隆（只需6秒参考音频）
tts.tts_to_file(
    text="Hello, this is a cloned voice speaking.",
    speaker_wav="reference.wav",
    language="en",
    file_path="cloned_output.wav",
)

5. 端到端语音理解

5.1 Qwen2-Audio

from transformers import Qwen2AudioForConditionalGeneration, AutoProcessor

model = Qwen2AudioForConditionalGeneration.from_pretrained(
    "Qwen/Qwen2-Audio-7B-Instruct", device_map="auto"
)
processor = AutoProcessor.from_pretrained("Qwen/Qwen2-Audio-7B-Instruct")

# 语音问答
conversation = [
    {"role": "user", "content": [
        {"type": "audio", "audio_url": "audio.wav"},
        {"type": "text", "text": "这段音频说了什么？"},
    ]},
]

inputs = processor.apply_chat_template(
    conversation, add_generation_prompt=True,
    tokenize=True, return_tensors="pt"
).to(model.device)

output = model.generate(**inputs, max_new_tokens=256)
response = processor.decode(output[0], skip_special_tokens=True)
print(response)

5.2 语音情感分析

def audio_emotion_analysis(audio_path):
    """语音情感分析"""
    conversation = [
        {"role": "user", "content": [
            {"type": "audio", "audio_url": audio_path},
            {"type": "text", "text": "分析这段音频中说话人的情感状态。"},
        ]},
    ]

    inputs = processor.apply_chat_template(
        conversation, add_generation_prompt=True,
        tokenize=True, return_tensors="pt"
    ).to(model.device)

    output = model.generate(**inputs, max_new_tokens=256)
    return processor.decode(output[0], skip_special_tokens=True)

6. 实时语音对话系统

import pyaudio
import numpy as np
import threading
import queue

class VoiceAssistant:
    """实时语音助手"""

    def __init__(self):
        self.asr_model = whisper.load_model("large-v3")
        self.tts_model = CosyVoice("CosyVoice-300M")
        self.audio_queue = queue.Queue()
        self.is_listening = False

    def start_listening(self):
        """开始录音"""
        self.is_listening = True
        stream = pyaudio.PyAudio().open(
            format=pyaudio.paFloat32,
            channels=1,
            rate=16000,
            input=True,
            frames_per_buffer=1024,
        )

        while self.is_listening:
            data = stream.read(1024)
            self.audio_queue.put(np.frombuffer(data, dtype=np.float32))

    def process_audio(self):
        """处理音频并生成回复"""
        audio_buffer = []

        while True:
            chunk = self.audio_queue.get()
            audio_buffer.extend(chunk)

            # 检测语音结束（静音检测）
            if len(audio_buffer) > 16000 * 3:  # 至少3秒
                audio_array = np.array(audio_buffer, dtype=np.float32)

                # 语音识别
                result = self.asr_model.transcribe(audio_array)
                user_text = result["text"]

                if user_text.strip():
                    print(f"用户: {user_text}")

                    # LLM 生成回复
                    response = self.get_llm_response(user_text)
                    print(f"助手: {response}")

                    # 语音合成
                    tts_output = self.tts_model.inference_sft(response, "中文女")
                    # 播放音频...

                audio_buffer = []

    def get_llm_response(self, text):
        """调用 LLM 生成回复"""
        from openai import OpenAI
        client = OpenAI()
        response = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": text}],
        )
        return response.choices[0].message.content

7. 总结

语音大模型的核心技术栈：

1. ASR（语音识别）：Whisper large-v3 是目前最好的开源方案
2. TTS（语音合成）：CosyVoice、Bark 提供高质量语音合成
3. 语音克隆：GPT-SoVITS 只需几秒参考音频即可克隆
4. 端到端理解：Qwen2-Audio 实现语音直接理解，无需先转文字