Whisper+GPT-SoVITS：语音识别到音色克隆的全链路实战

Whisper+GPT-SoVITS：语音识别到音色克隆的全链路实战

一、引言

语音 AI 正在经历爆发式增长。OpenAI 开源的 Whisper 实现了接近人类的语音识别能力，而 GPT-SoVITS 等音色克隆技术让"复制一个人的声音"成为现实。

本文将打通语音全链路：Whisper语音识别→微调→流式ASR→GPT-SoVITS音色克隆→端到端语音对话系统。

二、Whisper 语音识别

2.1 模型选择

模型	参数	显存	速度	中文识别率
tiny	39M	1GB	10x	85%
base	74M	1GB	7x	90%
small	244M	2GB	4x	94%
medium	769M	5GB	2x	96%
large-v3	1550M	10GB	1x	97%

2.2 基础使用

import whisper

model = whisper.load_model("large-v3")

# 1. 语音转文字
result = model.transcribe(
    "meeting.mp3",
    language="zh",
    task="transcribe",  # 或 "translate"（翻译为英文）
    verbose=True,
    # 高级参数
    temperature=0.0,
    beam_size=5,
    best_of=5,
    fp16=True,
    word_timestamps=True,     # 词级别时间戳
    condition_on_previous_text=True,
    initial_prompt="这是一场技术会议，讨论关于AI和机器学习。"
)

print(f"识别结果: {result['text']}")
print(f"语言: {result['language']}")

# 词级别时间戳
for segment in result["segments"]:
    print(f"[{segment['start']:.1f}s - {segment['end']:.1f}s] {segment['text']}")

2.3 中文微调

import torch
from transformers import (
    WhisperForConditionalGeneration,
    WhisperProcessor,
    Seq2SeqTrainingArguments,
    Seq2SeqTrainer
)
from datasets import load_dataset, Audio

# 微调 Whisper 适应领域术语
def fine_tune_whisper(custom_dataset_path, output_dir="./whisper-finetuned"):
    model_name = "openai/whisper-large-v3"
    model = WhisperForConditionalGeneration.from_pretrained(model_name)
    processor = WhisperProcessor.from_pretrained(model_name)
    
    # 加载自定义数据集
    dataset = load_dataset("json", data_files=custom_dataset_path)
    
    def preprocess(batch):
        # 加载音频
        audio = batch["audio"]
        # 提取特征
        features = processor(
            audio["array"],
            sampling_rate=16000,
            return_tensors="pt"
        ).input_features
        # 编码文本
        labels = processor.tokenizer(batch["text"]).input_ids
        return {"input_features": features, "labels": labels}
    
    dataset = dataset.map(preprocess)
    
    training_args = Seq2SeqTrainingArguments(
        output_dir=output_dir,
        per_device_train_batch_size=8,
        gradient_accumulation_steps=2,
        learning_rate=1e-5,
        warmup_steps=500,
        max_steps=4000,
        fp16=True,
        logging_steps=50,
        save_steps=500,
        evaluation_strategy="steps",
        eval_steps=500,
        predict_with_generate=True,
        generation_max_length=225,
    )
    
    trainer = Seq2SeqTrainer(
        model=model,
        args=training_args,
        train_dataset=dataset["train"],
        eval_dataset=dataset["test"],
        tokenizer=processor.feature_extractor,
    )
    
    trainer.train()
    model.save_pretrained(output_dir)
    processor.save_pretrained(output_dir)

2.4 实时流式 ASR

import numpy as np
import sounddevice as sd
import queue
import threading
import whisper

class StreamingASR:
    """实时流式语音识别"""
    
    def __init__(self, model_size="medium"):
        self.model = whisper.load_model(model_size)
        self.audio_queue = queue.Queue()
        self.sample_rate = 16000
        self.is_recording = False
        self.buffer = np.array([], dtype=np.float32)
        
    def audio_callback(self, indata, frames, time, status):
        """麦克风回调"""
        self.audio_queue.put(indata.copy())
    
    def start_recording(self):
        """开始录制"""
        self.is_recording = True
        self.stream = sd.InputStream(
            samplerate=self.sample_rate,
            channels=1,
            callback=self.audio_callback,
            blocksize=1600  # 100ms 块
        )
        self.stream.start()
        
        # 后台转写线程
        self.transcribe_thread = threading.Thread(target=self._transcribe_loop)
        self.transcribe_thread.start()
    
    def _transcribe_loop(self):
        """持续转写"""
        while self.is_recording:
            if not self.audio_queue.empty():
                chunk = self.audio_queue.get()
                self.buffer = np.concatenate([self.buffer, chunk.flatten()])
                
                # 每 3 秒转写一次
                if len(self.buffer) > 3 * self.sample_rate:
                    result = self.model.transcribe(
                        self.buffer,
                        language="zh",
                        task="transcribe"
                    )
                    print(f"实时转写: {result['text']}")
                    
                    # 保留最后 1 秒作为上下文
                    context_len = self.sample_rate
                    self.buffer = self.buffer[-context_len:]
    
    def stop_recording(self):
        """停止录制"""
        self.is_recording = False
        self.stream.stop()
        self.transcribe_thread.join()

asr = StreamingASR()
asr.start_recording()
# ... 说话中 ...
asr.stop_recording()

三、GPT-SoVITS 语音合成与克隆

3.1 环境搭建

git clone https://github.com/RVC-Boss/GPT-SoVITS.git
cd GPT-SoVITS
conda create -n sovits python=3.9 -y && conda activate sovits
pip install -r requirements.txt
pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu118

3.2 音色克隆流程

1. 准备参考音频（3-10秒清晰录音）
2. 音频预处理（降噪、音量归一化）
3. GPT阶段：文本→语义Token
4. SoVITS阶段：语义Token+参考音频→波形

import torch
import torchaudio
import numpy as np
from TTS_infer_pack.text_segmentation import split_sentences

class VoiceClone:
    def __init__(self, gpt_path, sovits_path):
        self.gpt_model = torch.load(gpt_path)
        self.sovits_model = torch.load(sovits_path)
        self.sample_rate = 32000
    
    def preprocess_audio(self, audio_path):
        """音频预处理"""
        waveform, sr = torchaudio.load(audio_path)
        # 重采样到32kHz
        if sr != 32000:
            resampler = torchaudio.transforms.Resample(sr, 32000)
            waveform = resampler(waveform)
        # 去噪
        waveform = torchaudio.functional.gain(waveform, 1.0)
        return waveform
    
    def extract_timbre(self, ref_audio_path, ref_text):
        """从参考音频提取音色特征"""
        ref_wav = self.preprocess_audio(ref_audio_path)
        # 音色编码
        with torch.no_grad():
            timbre = get_timbre_encoder()(ref_wav)
        return timbre
    
    def synthesize(self, text, timbre, ref_audio=None):
        """合成语音"""
        # 1. GPT阶段：文本→语义Token
        semantic_tokens = get_generate_model()(
            text, timbre, top_k=5, top_p=1.0, temperature=1.0
        )
        
        # 2. SoVITS阶段：Token+音色→波形
        if ref_audio is not None:
            audio = get_vits_model()(
                semantic_tokens, ref_audio
            )
        else:
            audio = get_vits_model()(
                semantic_tokens, timbre
            )
        
        return audio
    
    def text_to_speech(self, text, ref_audio_path, output_path):
        """完整TTS流程"""
        # 分句
        sentences = split_sentences(text)
        
        # 提取音色
        ref_text = "参考音频对应的文本"
        timbre = self.extract_timbre(ref_audio_path, ref_text)
        ref_wav = self.preprocess_audio(ref_audio_path)
        
        # 逐句合成
        audio_segments = []
        for sentence in sentences:
            audio = self.synthesize(sentence, timbre, ref_wav)
            audio_segments.append(audio)
        
        # 拼接所有片段
        full_audio = torch.cat(audio_segments, dim=-1)
        torchaudio.save(output_path, full_audio, self.sample_rate)
        
        return full_audio

四、传统TTS合集

4.1 Coqui TTS（快速上手）

import torch
from TTS.api import TTS

# 获取可用模型
tts = TTS(model_name="tts_models/zh-CN/baker/tacotron2-DDC-GST")
tts.tts_to_file(text="你好，欢迎使用语音合成技术。", file_path="output.wav")

# 多说话人模型
tts_multispeaker = TTS("tts_models/multilingual/multi-dataset/xtts_v2")
tts_multispeaker.tts_to_file(
    text="Hello, this is a voice clone demo.",
    speaker_wav="reference.wav",
    language="en",
    file_path="cloned_output.wav"
)

4.2 ChatTTS（对话风格）

import ChatTTS
from IPython.display import Audio

chat = ChatTTS.Chat()
chat.load_models()

# 带情绪控制的合成
texts = [
    "哇！这个消息太令人惊喜了！[laughter]",
    "嗯，让我想想这个问题怎么回答...",
    "请注意，前方路段有事故，请减速慢行。"
]

wavs = chat.infer(texts, use_decoder=True, params_infer_code={
    "spk_emb": None,  # 随机音色
    "temperature": 0.3,
    "top_P": 0.7,
    "top_K": 20,
})

for i, wav in enumerate(wavs):
    ChatTTS.tools.save_wav(wav, f"chattts_{i}.wav", 24000)

4.3 Fish Audio（最简方案）

from fish_audio_sdk import Session, TTSRequest

session = Session("your-api-key")

with open("output.mp3", "wb") as f:
    for chunk in session.tts(TTSRequest(
        text="用最自然的语气说出这句话。",
        reference_id="speaker-id"
    )):
        f.write(chunk)

五、语音到语音对话系统

import asyncio
import numpy as np

class VoiceChat:
    """语音对话系统：ASR → LLM → TTS"""
    
    def __init__(self):
        self.asr = whisper.load_model("medium")
        self.llm = ChatOpenAI(model="gpt-4")
        self.tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2")
        self.speaker_audio = "my_voice.wav"
    
    async def process(self, audio_input: np.ndarray) -> np.ndarray:
        # 1. 语音识别
        result = self.asr.transcribe(audio_input, language="zh")
        user_text = result["text"]
        print(f"👤 用户: {user_text}")
        
        # 2. LLM 生成回复
        response = await self.llm.achat(user_text)
        print(f"🤖 助手: {response}")
        
        # 3. 语音合成
        wav = self.tts.tts(
            text=response,
            speaker_wav=self.speaker_audio,
            language="zh"
        )
        
        return wav
    
    async def realtime_chat(self):
        """实时语音对话"""
        stream = pyaudio.PyAudio().open(
            format=pyaudio.paInt16, channels=1,
            rate=16000, input=True, frames_per_buffer=1024
        )
        
        while True:
            # 语音活动检测 (VAD)
            frames = []
            silent_count = 0
            
            while silent_count < 30:  # 1.5秒静默后停止
                data = stream.read(1024)
                audio_chunk = np.frombuffer(data, dtype=np.int16)
                
                # 简单能量检测
                energy = np.sqrt(np.mean(audio_chunk.astype(np.float32)**2))
                if energy > 500:
                    frames.append(audio_chunk)
                    silent_count = 0
                else:
                    if frames:
                        silent_count += 1
            
            if frames:
                audio = np.concatenate(frames).astype(np.float32) / 32768.0
                response_audio = await self.process(audio)
                # 播放回复
                sd.play(response_audio, 24000)
                sd.wait()

六、主流模型对比

特性	Whisper	GPT-SoVITS	ChatTTS	XTTS v2
语音识别	⭐⭐⭐⭐⭐	❌	❌	❌
音色克隆	❌	⭐⭐⭐⭐⭐	⭐⭐	⭐⭐⭐⭐
情绪控制	❌	⭐⭐	⭐⭐⭐⭐	⭐⭐⭐
多语言	99+	中英日	中英	17语言
实时推理	✅	✅	✅	✅
开源	✅	✅	✅	✅

七、总结

语音AI全链路的核心组件：

1. Whisper --- 最强开源ASR，large-v3识别率达97%
2. GPT-SoVITS --- 3秒音频即可克隆音色，创造性最强
3. ChatTTS --- 对话风格合成，情绪自然
4. XTTS v2 --- 多语言支持最好，适合国际化场景

组合使用可实现完整的语音对话系统：Whisper 听 → LLM 想 → GPT-SoVITS 说。