提交

backend/ai_service.py

@@ -262,144 +262,116 @@ Great job! That's the correct pronunciation. Keep practicing and you'll get even
         self, text: str, emotion: EmotionType = "neutral"
     ) -> AsyncGenerator[bytes, None]:
         """
-        文本转语音（流式）
+        文本转语音（流式 - 实时返回音频块）
         
         Args:
             text: 要合成的文本
             emotion: 情感类型
             
         Yields:
-            audio_data: 音频数据块
+            audio_data: PCM 音频数据块（Base64 解码后的原始数据）
         """
         try:
-            print(f"\n========== TTS 开始 ==========")
+            import time
+            tts_start = time.time()
+            
+            print(f"\n========== TTS 流式开始 ==========")
             print(f"要合成的文本: {text}")
             print(f"情感: {emotion}")
             
-            # 根据情感选择音色或参数
+            # 根据情感选择音色
             voice = self._get_voice_for_emotion(emotion)
             print(f"使用音色: {voice}")
             print(f"TTS模型: {settings.tts_model}")
+            print(f"语速: {settings.tts_speech_rate}x (1.0=正常)")
+            print(f"流式模式: 是 (PCM 24kHz)")
             
-            # 在线程池中执行同步调用，避免事件循环冲突
-            # 添加重试机制以应对网络不稳定
-            def sync_tts():
-                max_retries = 3
-                retry_delay = 1  # 秒
-                
-                for attempt in range(max_retries):
-                    try:
-                        return dashscope.audio.qwen_tts.SpeechSynthesizer.call(
-                            model=settings.tts_model,
-                            text=text,
-                            voice=voice,
-                        )
-                    except Exception as e:
-                        if attempt < max_retries - 1:
-                            print(f"TTS调用失败(尝试 {attempt + 1}/{max_retries}): {e}")
-                            print(f"等待{retry_delay}秒后重试...")
-                            import time
-                            time.sleep(retry_delay)
-                            retry_delay *= 2  # 指数退避
-                        else:
-                            raise
+            # 使用流式 API
+            def sync_streaming_tts():
+                return dashscope.MultiModalConversation.call(
+                    api_key=settings.dashscope_api_key,
+                    model=settings.tts_model,
+                    text=text,
+                    voice=voice,
+                    speech_rate=settings.tts_speech_rate,  # 语速调整
+                    stream=True  # 🔥 启用流式
+                )
             
-            print("调用 DashScope TTS API...")
-            # 调用TTS API
-            response = await asyncio.to_thread(sync_tts)
+            print("调用 DashScope TTS 流式 API...")
             
-            print(f"TTS API 响应类型: {type(response)}")
+            # 在线程池中调用流式 API
+            response = await asyncio.to_thread(sync_streaming_tts)
             
-            # 获取音频数据
-            # response.output 包含 audio 字典，其中有 data、url、id、expires_at
-            audio_bytes = None
+            chunk_count = 0
+            first_chunk_time = None
+            total_audio_size = 0
             
-            if hasattr(response, 'output'):
-                output = response.output
-                print(f"output 类型: {type(output)}")
-                
-                # 打印finish_reason用于调试
-                if isinstance(output, dict):
-                    print(f"finish_reason: {output.get('finish_reason', 'N/A')}")
-                    audio_info = output.get('audio', {})
-                elif hasattr(output, 'finish_reason'):
-                    print(f"finish_reason: {output.finish_reason}")
-                    audio_info = output.get('audio', {}) if isinstance(output, dict) else getattr(output, 'audio', {})
-                else:
-                    audio_info = {}
-                
-                # 检查是否有URL（新版API返回URL）
-                if isinstance(audio_info, dict) and 'url' in audio_info and audio_info['url']:
-                    audio_url = audio_info['url']
-                    print(f"从URL下载音频: {audio_url}")
+            # 流式接收音频数据
+            for chunk in response:
+                if hasattr(chunk, 'output') and hasattr(chunk.output, 'audio'):
+                    audio = chunk.output.audio
                     
-                    # 下载音频数据
-                    import aiohttp
-                    async with aiohttp.ClientSession() as session:
-                        async with session.get(audio_url) as resp:
-                            if resp.status == 200:
-                                audio_bytes = await resp.read()
-                                print(f"音频下载成功，大小: {len(audio_bytes)} 字节")
-                            else:
-                                raise Exception(f"下载音频失败，HTTP状态码: {resp.status}")
-                
-                # 检查是否有直接的数据（旧版API）
-                elif isinstance(audio_info, dict) and 'data' in audio_info and audio_info['data']:
-                    audio_data = audio_info['data']
-                    if isinstance(audio_data, bytes):
-                        audio_bytes = audio_data
-                        print(f"音频数据(bytes): {len(audio_bytes)} 字节")
-                    elif isinstance(audio_data, str):
-                        audio_bytes = base64.b64decode(audio_data)
-                        print(f"音频数据(base64解码): {len(audio_bytes)} 字节")
-                
-                # 如果audio_info直接是bytes或str
-                elif isinstance(audio_info, bytes):
-                    audio_bytes = audio_info
-                    print(f"音频数据(bytes): {len(audio_bytes)} 字节")
-                elif isinstance(audio_info, str):
-                    audio_bytes = base64.b64decode(audio_info)
-                    print(f"音频数据(base64解码): {len(audio_bytes)} 字节")
+                    # 接收到音频数据块
+                    if audio.data is not None:
+                        # 记录首字节时间
+                        if first_chunk_time is None:
+                            first_chunk_time = time.time()
+                            first_byte_latency = (first_chunk_time - tts_start) * 1000
+                            print(f"⚡ 首字节延迟: {first_byte_latency:.1f}ms")
+                        
+                        # Base64 解码 PCM 数据
+                        pcm_bytes = base64.b64decode(audio.data)
+                        chunk_count += 1
+                        total_audio_size += len(pcm_bytes)
+                        
+                        # 立即返回给调用方（实时传输）
+                        yield pcm_bytes
+                        await asyncio.sleep(0)  # 让出控制权
+                    
+                    # 检查是否结束
+                    if chunk.output.finish_reason == "stop":
+                        tts_end = time.time()
+                        total_duration = (tts_end - tts_start) * 1000
+                        
+                        print(f"✅ TTS 流式完成:")
+                        print(f"   - 总块数: {chunk_count}")
+                        print(f"   - 音频大小: {total_audio_size} 字节")
+                        print(f"   - 总耗时: {total_duration:.0f}ms")
+                        if first_chunk_time:
+                            print(f"   - 首字节延迟: {(first_chunk_time - tts_start) * 1000:.0f}ms")
+                        
+                        # 完整音频 URL（可选，用于备份）
+                        if hasattr(audio, 'url') and audio.url:
+                            print(f"   - 完整音频 URL: {audio.url}")
+                        if hasattr(audio, 'expires_at') and audio.expires_at:
+                            print(f"   - 过期时间: {audio.expires_at}")
+                        
+                        print(f"========== TTS 流式成功 ==========\n")
+                        break
             
-            if audio_bytes:
-                
-                # 将音频数据分块返回
-                chunk_size = 3200  # 每块3200字节
-                chunk_count = (len(audio_bytes) + chunk_size - 1) // chunk_size
-                print(f"将分为 {chunk_count} 个数据块发送")
-                
-                for i in range(0, len(audio_bytes), chunk_size):
-                    chunk = audio_bytes[i:i + chunk_size]
-                    yield chunk
-                    await asyncio.sleep(0)  # 让出控制权
-                
-                print(f"========== TTS 成功 ==========\n")
-            else:
-                error_msg = "TTS API未返回音频数据"
-                print(f"错误: {error_msg}")
-                print(f"========== TTS 失败 ==========\n")
-                raise Exception(error_msg)
+            if chunk_count == 0:
+                raise Exception("TTS API 未返回任何音频数据")
                 
         except Exception as e:
-            print(f"========== TTS 错误 ==========")
+            print(f"========== TTS 流式错误 ==========")
             print(f"错误: {e}")
             import traceback
             traceback.print_exc()
-            print(f"========== TTS 错误结束 ==========\n")
+            print(f"========== TTS 流式错误结束 ==========\n")
             raise
     
     def _get_voice_for_emotion(self, emotion: EmotionType) -> str:
         """根据情感选择音色"""
-        # 情感到音色的映射
+        # 情感到音色的映射（统一使用 Elias 男声）
         voice_map = {
-            "neutral": "Cherry",
-            "happy": "Cherry",
-            "sad": "Cherry",
-            "excited": "Cherry",
-            "angry": "Cherry",
-            "surprised": "Cherry",
-            "thoughtful": "Cherry",
-            "encouraging": "Cherry",
+            "neutral": "Elias",
+            "happy": "Elias",
+            "sad": "Elias",
+            "excited": "Elias",
+            "angry": "Elias",
+            "surprised": "Elias",
+            "thoughtful": "Elias",
+            "encouraging": "Elias",
         }
         return voice_map.get(emotion, settings.tts_voice)
 

backend/config.py

@@ -35,7 +35,8 @@ class Settings(BaseSettings):
     
     # TTS配置
     tts_model: str = os.getenv("TTS_MODEL", "qwen3-tts-flash")
-    tts_voice: str = os.getenv("TTS_VOICE", "Cherry")
+    tts_voice: str = os.getenv("TTS_VOICE", "Elias")  # 男声
+    tts_speech_rate: float = float(os.getenv("TTS_SPEECH_RATE", "1.5"))  # 语速：1.2 = 加快20%
     
     # 豆包API超时配置
     DOUBAO_RECV_TIMEOUT: int = 15  # 接收消息超时（秒）

backend/models.py

@@ -95,9 +95,10 @@ class AudioResponseMessage(BaseMessage):
     """音频响应消息"""
     type: Literal["audio_response"] = "audio_response"
     data: str
-    sampleRate: int = 16000
+    sampleRate: int = 24000  # 流式 TTS 使用 24kHz
     channels: int = 1
     bitDepth: int = 16
+    format: Literal["pcm", "wav"] = "pcm"  # 音频格式：pcm（流式）或 wav（完整）
     chunkIndex: int
     isLast: bool
 

backend/websocket_handler.py

@@ -404,7 +404,8 @@ class WebSocketHandler:
         await self.websocket.send_text(message.model_dump_json())
     
     async def send_audio_response(
-        self, request_id: str, data: str, chunk_index: int, is_last: bool
+        self, request_id: str, data: str, chunk_index: int, is_last: bool,
+        format: str = "pcm", sample_rate: int = 24000
     ):
         """发送音频响应"""
         message = AudioResponseMessage(
@@ -412,6 +413,8 @@ class WebSocketHandler:
             requestId=request_id,
             timestamp=self._get_timestamp(),
             data=data,
+            sampleRate=sample_rate,
+            format=format,
             chunkIndex=chunk_index,
             isLast=is_last,
         )

frontend/src/App.tsx

@@ -102,15 +102,25 @@ function App() {
           setState('playing');
         }
         
-        // Base64 解码音频数据（WAV 格式）
-        const audioData = atob(message.data);
-        const wavData = new Uint8Array(audioData.length);
-        
-        for (let i = 0; i < audioData.length; i++) {
-          wavData[i] = audioData.charCodeAt(i);
+        // 处理音频数据（支持 PCM 和 WAV 格式）
+        if (message.data) {
+          // Base64 解码音频数据
+          const audioData = atob(message.data);
+          const audioBytes = new Uint8Array(audioData.length);
+          
+          for (let i = 0; i < audioData.length; i++) {
+            audioBytes[i] = audioData.charCodeAt(i);
+          }
+          
+          // 传递采样率参数（PCM 格式需要）
+          playerRef.current?.addChunk(audioBytes, message.sampleRate || 24000);
         }
         
-        playerRef.current?.addChunk(wavData);
+        // 检查是否为最后一块
+        if (message.isLast) {
+          console.log('收到最后一块音频，标记流结束');
+          playerRef.current?.markStreamEnd();
+        }
         break;
 
       case 'error':

frontend/src/types/index.ts

@@ -95,9 +95,10 @@ export interface EmotionMessage extends BaseMessage {
 export interface AudioResponseMessage extends BaseMessage {
   type: 'audio_response';
   data: string;
-  sampleRate: 16000;
+  sampleRate: number; // 24000 for PCM streaming
   channels: 1;
   bitDepth: 16;
+  format: 'pcm' | 'wav'; // Audio format
   chunkIndex: number;
   isLast: boolean;
 }

frontend/src/utils/audioPlayer.ts

@@ -1,98 +1,142 @@
 /**
  * 音频播放器
- * 支持 WAV 文件流式播放
+ * 支持 PCM 流式播放（24kHz, 16bit, Mono）
  */
 export class AudioPlayer {
   private audioContext: AudioContext | null = null;
-  private audioChunks: Uint8Array[] = []; // 收集音频块
   private isPlaying = false;
-  private currentSource: AudioBufferSourceNode | null = null;
   private onEndCallback: (() => void) | null = null;
+  
+  // 流式播放相关
+  private audioQueue: AudioBuffer[] = []; // 音频缓冲队列
+  private isProcessingQueue = false;
+  private nextStartTime = 0; // 下一个音频块的开始时间
+  private currentSources: AudioBufferSourceNode[] = []; // 当前播放的音频源
 
   async initialize(): Promise<void> {
-    this.audioContext = new AudioContext();
+    // 创建 AudioContext，采样率 24000Hz（与后端 TTS 一致）
+    this.audioContext = new AudioContext({ sampleRate: 24000 });
+    console.log(`AudioPlayer 初始化 - 采样率: ${this.audioContext.sampleRate}Hz`);
   }
 
   /**
-   * 添加音频数据块（WAV 格式）
+   * 添加 PCM 音频数据块（流式）
+   * @param data PCM 原始数据（Int16Array 的字节数组）
+   * @param sampleRate 采样率（默认 24000Hz）
    */
-  async addChunk(data: Uint8Array): Promise<void> {
+  async addChunk(data: Uint8Array, sampleRate: number = 24000): Promise<void> {
     if (!this.audioContext) {
       throw new Error('AudioPlayer未初始化');
     }
 
-    // 跳过空数据块（用于标记结束）
+    // 跳过空数据块（用于标记句子结束）
     if (data.length === 0) {
-      // 收到结束标记，播放收集的所有数据
-      await this.playCollectedData();
+      console.log('收到空块，继续等待更多音频...');
       return;
     }
 
-    // 收集音频块
-    this.audioChunks.push(data);
+    // 将 PCM 字节数据转换为 Int16Array
+    const pcmData = new Int16Array(data.buffer);
+    
+    // 创建 AudioBuffer
+    const audioBuffer = this.audioContext.createBuffer(
+      1, // 单声道
+      pcmData.length,
+      sampleRate
+    );
+    
+    // 将 PCM 数据转换为 Float32（-1.0 到 1.0）
+    const channelData = audioBuffer.getChannelData(0);
+    for (let i = 0; i < pcmData.length; i++) {
+      channelData[i] = pcmData[i] / 32768.0; // Int16 转 Float32
+    }
+    
+    // 添加到播放队列
+    this.audioQueue.push(audioBuffer);
+    
+    // 立即开始处理队列（流式播放）
+    this.processQueue();
   }
 
   /**
-   * 播放收集的音频数据
+   * 处理音频队列（实现无缝播放）
    */
-  private async playCollectedData(): Promise<void> {
-    if (!this.audioContext || this.audioChunks.length === 0) {
+  private processQueue(): void {
+    if (this.isProcessingQueue || this.audioQueue.length === 0 || !this.audioContext) {
       return;
     }
 
-    try {
-      // 合并所有音频块
-      const totalLength = this.audioChunks.reduce((sum, chunk) => sum + chunk.length, 0);
-      const mergedData = new Uint8Array(totalLength);
-      
-      let offset = 0;
-      for (const chunk of this.audioChunks) {
-        mergedData.set(chunk, offset);
-        offset += chunk.length;
-      }
+    this.isProcessingQueue = true;
+    this.isPlaying = true;
 
-      // 使用 Web Audio API 自动解码 WAV 数据
-      const audioBuffer = await this.audioContext.decodeAudioData(mergedData.buffer);
-      
-      console.log(`音频已解码 - 采样率: ${audioBuffer.sampleRate}Hz, 时长: ${audioBuffer.duration.toFixed(2)}s`);
+    // 获取当前时间
+    const currentTime = this.audioContext.currentTime;
+    
+    // 如果 nextStartTime 还未初始化或已过期，重置为当前时间
+    if (this.nextStartTime < currentTime) {
+      this.nextStartTime = currentTime;
+    }
 
-      // 播放
-      this.isPlaying = true;
+    // 播放队列中的所有音频块
+    while (this.audioQueue.length > 0) {
+      const audioBuffer = this.audioQueue.shift()!;
+      
       const source = this.audioContext.createBufferSource();
       source.buffer = audioBuffer;
       source.connect(this.audioContext.destination);
       
+      // 在队列中的下一个时间点开始播放（实现无缝衔接）
+      source.start(this.nextStartTime);
+      
+      // 更新下一个音频块的开始时间
+      this.nextStartTime += audioBuffer.duration;
+      
+      // 保存音频源引用
+      this.currentSources.push(source);
+      
+      // 当这个音频块播放完成时，从数组中移除
       source.onended = () => {
-        this.isPlaying = false;
-        if (this.onEndCallback) {
-          this.onEndCallback();
+        const index = this.currentSources.indexOf(source);
+        if (index > -1) {
+          this.currentSources.splice(index, 1);
+        }
+        
+        // 如果所有音频都播放完了，触发结束回调
+        if (this.currentSources.length === 0 && this.audioQueue.length === 0) {
+          this.isPlaying = false;
+          if (this.onEndCallback) {
+            this.onEndCallback();
+          }
         }
       };
-
-      source.start(0);
-      this.currentSource = source;
-
-      // 清空缓存
-      this.audioChunks = [];
-    } catch (error) {
-      console.error('解码音频失败:', error);
-      throw error;
     }
+
+    this.isProcessingQueue = false;
   }
 
+  /**
+   * 标记音频流结束
+   */
+  markStreamEnd(): void {
+    console.log('音频流结束标记');
+    // 确保队列中的所有音频都被播放
+    this.processQueue();
+  }
 
   stop(): void {
-    if (this.currentSource) {
+    // 停止所有正在播放的音频
+    this.currentSources.forEach(source => {
       try {
-        this.currentSource.stop();
+        source.stop();
       } catch (e) {
         // 忽略已停止的错误
       }
-      this.currentSource = null;
-    }
-
-    this.audioChunks = [];
+    });
+    
+    this.currentSources = [];
+    this.audioQueue = [];
     this.isPlaying = false;
+    this.nextStartTime = 0;
   }
 
   onEnd(callback: () => void): void {