提交

backend/src/config/doubao_config.py

@@ -44,8 +44,11 @@ start_session_req = {
         "extra": {
             "strict_audit": False,
             "audit_response": "支持客户自定义安全审核回复话术。",
-            "recv_timeout": 10,
-            "input_mod": "audio"
+            "recv_timeout": 30,  # 增加超时时间，避免DialogAudioIdleTimeoutError
+            "input_mod": "audio",
+            "streaming_mode": True,  # 启用流式模式
+            "enable_vad": True,      # 启用语音活动检测
+            "vad_silence_threshold": 1000  # VAD静音阈值(ms)
         }
     }
 }

backend/src/services/message_router.py

@@ -537,20 +537,20 @@ class MessageRouter:
             
             silence_chunk = b'\x00' * 3200  # 3200字节静音数据 (100ms @ 16kHz)
             silence_count = 0
-            max_silence_duration = 30  # 最多30秒静音
+            max_silence_duration = 15  # 最多15秒静音，避免豆包服务器超时
             
             logger.info("开始发送静音数据保持连接", extra={
                 "session_id": session.session_id,
                 "max_duration": max_silence_duration
             })
             
-            while silence_count < max_silence_duration * 10:  # 每100ms一次
+            while silence_count < max_silence_duration * 5:  # 每200ms一次
                 # 检查是否还需要保持连接
                 if (hasattr(session, '_response_task') and 
                     session._response_task.done()):
                     logger.info("响应接收完成，停止发送静音数据", extra={
                         "session_id": session.session_id,
-                        "silence_duration": silence_count * 0.1
+                        "silence_duration": silence_count * 0.2
                     })
                     break
                 
@@ -566,16 +566,16 @@ class MessageRouter:
                 silence_count += 1
                 
                 # 每秒记录一次日志
-                if silence_count % 10 == 0:
+                if silence_count % 5 == 0:
                     logger.debug("保持连接中", extra={
                         "session_id": session.session_id,
-                        "silence_duration": silence_count / 10
+                        "silence_duration": silence_count / 5
                     })
                 
-                # 等待100ms
-                await asyncio.sleep(0.1)
+                # 等待200ms，减少发送频率，降低服务器压力
+                await asyncio.sleep(0.2)
             
-            if silence_count >= max_silence_duration * 10:
+            if silence_count >= max_silence_duration * 5:
                 logger.warning("静音发送超时", extra={
                     "session_id": session.session_id,
                     "max_duration": max_silence_duration
@@ -782,7 +782,7 @@ class MessageRouter:
                     if content:
                         logger.info(f"AI文本响应: {content}", extra={"session_id": session.session_id})
                         # 发送AI文本响应到WebSocket客户端
-                        await self._send_to_websocket(session, {
+                        await self._send_to_client(session.websocket, {
                             "type": "ai_text_response",
                             "content": content,
                             "timestamp": datetime.now().isoformat()

frontend/src/App.tsx

@@ -163,7 +163,7 @@ function App() {
     connect,
     disconnect,
     sendAudioBlob,
-    startSession,
+    sendRawPCMAudio,
     isConnected: wsConnected,
     connectionStatus
   } = useWebSocket({
@@ -332,16 +332,9 @@ function App() {
 
       // 检查会话状态
       if (!session?.id) {
-        console.log('[App] 没有活动会话，开始创建会话');
-        try {
-          await startSession('zh-CN');
-          // 等待会话建立
-          await new Promise(resolve => setTimeout(resolve, 1000));
-        } catch (error) {
-          console.error('[App] 创建会话失败:', error);
-          dispatch(setError('无法建立会话，请重试'));
-          return;
-        }
+        console.log('[App] 没有活动会话，等待会话建立');
+        dispatch(setError('请等待会话建立后再开始录音'));
+        return;
       }
 
       // 发送音频数据
@@ -364,7 +357,43 @@ function App() {
         duration: 4,
       });
     }
-  }, [wsConnected, session, startSession, sendAudioBlob, dispatch]);
+  }, [wsConnected, session, sendAudioBlob, dispatch]);
+
+  // 处理实时音频数据
+  const handleRealtimeAudioData = useCallback(async (pcmData: Int16Array, chunkId: number) => {
+    try {
+      console.log(`[App] 收到实时音频数据: chunk ${chunkId}, samples: ${pcmData.length}, 前几个值: [${pcmData.slice(0, 5).join(', ')}]`);
+      
+      // 调试信息：检查连接和会话状态
+      console.log(`[App] 状态检查详细信息:`);
+      console.log(`  - Redux isConnected: ${isConnected}`);
+      console.log(`  - Hook wsConnected: ${wsConnected}`);
+      console.log(`  - Redux session: ${JSON.stringify(session)}`);
+      console.log(`  - Hook connectionStatus: ${connectionStatus}`);
+      
+      // 检查WebSocket连接和会话状态
+      if (!wsConnected || !session?.id) {
+        console.warn('[App] WebSocket未连接或会话无效，跳过实时音频发送', {
+          reduxIsConnected: isConnected,
+          hookWsConnected: wsConnected,
+          sessionId: session?.id,
+          sessionExists: !!session,
+          connectionStatus,
+          sessionObject: session
+        });
+        return;
+      }
+
+      console.log(`[App] 发送实时音频数据: chunk ${chunkId}, samples: ${pcmData.length}, size: ${pcmData.byteLength} bytes`);
+      
+      // 直接发送原始PCM数据，不需要转换
+      await sendRawPCMAudio(pcmData, false); // 不是最后一个块
+      
+    } catch (error) {
+      console.error('[App] 发送实时音频数据失败:', error);
+      // 不要在这里设置错误，因为这是实时数据，偶尔失败是正常的
+    }
+  }, [isConnected, wsConnected, session, sendRawPCMAudio, connectionStatus]);
 
   // 初始化WebSocket连接
   useEffect(() => {
@@ -383,23 +412,8 @@ function App() {
     initializeConnection();
   }, [wsConnected, dispatch]); // 移除connect依赖，避免无限循环
 
-  // 启动会话
-  useEffect(() => {
-    const initializeSession = async () => {
-      try {
-        if (wsConnected && !session) {
-          console.log('[App] 启动会话...');
-          await startSession('zh-CN'); // 使用中文会话
-          console.log('[App] 会话初始化请求已发送');
-        }
-      } catch (error) {
-        console.error('[App] 启动会话失败:', error);
-        dispatch(setError('启动会话失败'));
-      }
-    };
-    
-    initializeSession();
-  }, [wsConnected, session, dispatch]); // 移除startSession依赖，避免无限循环
+  // 会话状态监听 - 移除重复的会话初始化逻辑
+  // WebSocket连接后会自动发送session_init消息，无需在这里重复发送
 
   // 监听会话状态变化
   useEffect(() => {
@@ -435,14 +449,10 @@ function App() {
   // 监听连接状态变化
   useEffect(() => {
     if (wsConnected && !session) {
-      // 连接成功但没有会话，自动创建会话
-      console.log('[App] 连接成功，准备创建会话...');
-      startSession('zh-CN').catch(error => {
-        console.error('[App] 自动创建会话失败:', error);
-        dispatch(setError('无法建立会话，请刷新页面重试'));
-      });
+      // 连接成功但没有会话，等待会话自动建立
+      console.log('[App] 连接成功，等待会话自动建立...');
     }
-  }, [wsConnected, session, startSession, dispatch]);
+  }, [wsConnected, session, dispatch]);
 
   return (
     <ErrorBoundary>
@@ -502,6 +512,7 @@ function App() {
             
             <VoiceRecorder
               onRecordingComplete={handleRecordingComplete}
+              onRealtimeAudioData={handleRealtimeAudioData}
               disabled={!wsConnected || !session || recordingStatus === 'processing'}
             />
           </RecorderSection>

frontend/src/components/VoiceRecorder.tsx

@@ -18,6 +18,7 @@ declare global {
 
 interface VoiceRecorderProps {
   onRecordingComplete?: (audioBlob: Blob) => void;
+  onRealtimeAudioData?: (pcmData: Int16Array, chunkId: number) => void;
   disabled?: boolean;
 }
 
@@ -172,6 +173,7 @@ const DebugInfo = styled.div`
 
 const VoiceRecorder: React.FC<VoiceRecorderProps> = ({ 
   onRecordingComplete, 
+  onRealtimeAudioData,
   disabled = false 
 }) => {
   const dispatch = useDispatch();
@@ -183,6 +185,7 @@ const VoiceRecorder: React.FC<VoiceRecorderProps> = ({
     stopRecording, 
     cancelRecording,
     setRecordingCanceling,
+    setRealtimeAudioCallback,
     initialize,
     isSupported 
   } = useAudio();
@@ -226,6 +229,12 @@ const VoiceRecorder: React.FC<VoiceRecorderProps> = ({
         await initialize();
         setIsInitialized(true);
         addDebugLog('音频系统初始化成功');
+        
+        // 设置实时音频数据回调
+        if (onRealtimeAudioData) {
+          addDebugLog('设置实时音频数据回调');
+          setRealtimeAudioCallback(onRealtimeAudioData);
+        }
       } catch (error) {
         const errorMsg = error instanceof Error ? error.message : String(error);
         addDebugLog(`音频初始化失败: ${errorMsg}`);
@@ -244,7 +253,7 @@ const VoiceRecorder: React.FC<VoiceRecorderProps> = ({
     if (!isInitialized) {
       initAudio();
     }
-  }, [initialize, isInitialized, addDebugLog, isMobileDevice]);
+  }, [initialize, isInitialized, addDebugLog, isMobileDevice, onRealtimeAudioData, setRealtimeAudioCallback]);
 
   // 监听错误消息
   useEffect(() => {

frontend/src/hooks/useAudio.ts

@@ -62,14 +62,8 @@ export const useAudio = () => {
           await recorderRef.current.preInitialize();
           console.log('[useAudio] 优化录音器预初始化完成');
           
-          // 设置实时PCM数据回调
-          recorderRef.current.setOnAudioData((pcmData) => {
-            // 将PCM数据转换为Blob格式以兼容现有接口
-            const blob = new Blob([pcmData.buffer], { type: 'audio/pcm' });
-            dispatch(addAudioChunk(blob));
-            chunkCounterRef.current++;
-            console.debug(`[useAudio] 收到PCM数据: ${pcmData.length} samples`);
-          });
+          // 设置实时PCM数据回调 - 暂时不设置，等待外部回调设置时再统一处理
+          // 这样避免回调被覆盖的问题
           
           // 设置可视化回调
           recorderRef.current.setVisualizationCallback((data) => {
@@ -375,6 +369,28 @@ export const useAudio = () => {
     return cleanup;
   }, [cleanup]);
 
+  // 设置实时音频数据回调
+  const setRealtimeAudioCallback = useCallback((callback: (pcmData: Int16Array, chunkId: number) => void) => {
+    console.log('[useAudio] 设置实时音频数据回调');
+    if (recorderRef.current) {
+      let chunkId = 0;
+      recorderRef.current.setOnAudioData((pcmData) => {
+        chunkId++;
+        console.log(`[useAudio] 音频数据回调触发: chunk ${chunkId}, samples: ${pcmData.length}`);
+        
+        // 内部处理：更新Redux状态和可视化
+        const blob = new Blob([pcmData.buffer], { type: 'audio/pcm' });
+        dispatch(addAudioChunk(blob));
+        chunkCounterRef.current++;
+        
+        // 外部回调：发送到WebSocket
+        callback(pcmData, chunkId);
+      });
+    } else {
+      console.warn('[useAudio] 录制器未初始化，无法设置实时音频回调');
+    }
+  }, [dispatch]);
+
   // 返回Hook接口
   return {
     // 状态
@@ -392,6 +408,7 @@ export const useAudio = () => {
     stopPlayback,
     setAudioVolume,
     setRecordingCanceling,
+    setRealtimeAudioCallback,
     initialize,
     cleanup,
 

frontend/src/hooks/useWebSocket.ts

@@ -51,15 +51,21 @@ const getWebSocketUrl = (customUrl?: string): string => {
       }
     } else {
       // 自动协议检测
-      const isHttps = window.location.protocol === 'https:';
-      const protocol = isHttps ? 'wss:' : 'ws:';
       const hostname = window.location.hostname;
       
-      // 根据协议选择端口
-      const defaultPort = isHttps ? '8766' : '8765';
-      const port = process.env.REACT_APP_WS_PORT || defaultPort;
-      
-      baseUrl = `${protocol}//${hostname}:${port}`;
+      // 在localhost开发环境中强制使用WS协议
+      if (hostname === 'localhost' || hostname === '127.0.0.1') {
+        const protocol = 'ws:';
+        const port = process.env.REACT_APP_WS_PORT || '8765';
+        baseUrl = `${protocol}//${hostname}:${port}`;
+      } else {
+        // 生产环境使用HTTPS检测
+        const isHttps = window.location.protocol === 'https:';
+        const protocol = isHttps ? 'wss:' : 'ws:';
+        const defaultPort = isHttps ? '8766' : '8765';
+        const port = process.env.REACT_APP_WS_PORT || defaultPort;
+        baseUrl = `${protocol}//${hostname}:${port}`;
+      }
     }
   }
   
@@ -113,7 +119,7 @@ export const useWebSocket = (options: UseWebSocketOptions = {}) => {
         wsRef.current.onAudioResponse(options.onAudioResponse);
         console.log('[useWebSocket] 音频响应回调已设置');
       }
-      wsRef.current.onOpen(() => {
+      wsRef.current.onOpen(async () => {
         console.log('useWebSocket: WebSocket连接已建立，设置状态为已连接');
         connectingRef.current = false; // 重置连接状态
         dispatch(setConnected(true));
@@ -124,6 +130,17 @@ export const useWebSocket = (options: UseWebSocketOptions = {}) => {
           clearTimeout(errorTimeoutRef.current);
           errorTimeoutRef.current = null;
         }
+        
+        // 自动发送会话初始化消息
+        try {
+          console.log('[useWebSocket] 自动发送会话初始化消息');
+          await wsRef.current?.sendMessage({
+            type: 'session_init',
+            timestamp: Date.now() / 1000,
+          });
+        } catch (error) {
+          console.error('[useWebSocket] 发送会话初始化消息失败:', error);
+        }
       });
 
       wsRef.current.onClose(() => {
@@ -164,30 +181,50 @@ export const useWebSocket = (options: UseWebSocketOptions = {}) => {
             
             switch (message.type) {
               case 'session_init_response':
-                if (message.data?.session_id) {
-                  dispatch(setSession(message.data.session_id));
-                  console.log('[useWebSocket] 会话已初始化:', message.data.session_id);
-                }
-                break;
-              
-              case 'latency_response':
-                if (typeof message.data?.latency === 'number') {
-                  dispatch(setLatency(message.data.latency));
-                }
-                break;
-              
               case 'session_init':
                 console.log('[useWebSocket] 收到会话初始化响应:', message);
-                if (message.session_id) {
-                  dispatch(setSession({
+                // 处理会话初始化响应
+                let sessionData;
+                
+                // 优先检查直接在消息中的格式（当前后端使用的格式）
+                if (message.session_id && message.success) {
+                  sessionData = {
                     id: message.session_id,
                     userId: message.user_id,
                     language: message.language || 'zh-CN',
                     startTime: Date.now(),
                     lastActivity: Date.now(),
                     isActive: true,
-                  }));
-                  console.log('[useWebSocket] 会话状态已更新到Redux');
+                  };
+                  console.log('[useWebSocket] 使用直接格式创建会话数据:', sessionData);
+                  
+                } else if (message.data?.session_id) {
+                  // 备选格式：使用data字段
+                  sessionData = {
+                    id: message.data.session_id,
+                    userId: message.data.user_id,
+                    language: message.data.language || 'zh-CN',
+                    startTime: Date.now(),
+                    lastActivity: Date.now(),
+                    isActive: true,
+                  };
+                  console.log('[useWebSocket] 使用data字段格式创建会话数据:', sessionData);
+                } else {
+                  console.warn('[useWebSocket] 会话初始化响应格式不正确:', message);
+                }
+                
+                if (sessionData) {
+                  dispatch(setSession(sessionData));
+                  dispatch(setConnectionStatus('connected'));
+                  console.log('[useWebSocket] 会话状态已更新到Redux:', sessionData);
+                } else {
+                  console.error('[useWebSocket] 无法创建会话数据，消息格式:', message);
+                }
+                break;
+              
+              case 'latency_response':
+                if (typeof message.data?.latency === 'number') {
+                  dispatch(setLatency(message.data.latency));
                 }
                 break;
               
@@ -278,6 +315,21 @@ export const useWebSocket = (options: UseWebSocketOptions = {}) => {
     }
   }, [isConnected]);
 
+  // 发送原始PCM音频数据（跳过转换）
+  const sendRawPCMAudio = useCallback(async (pcmData: Int16Array, isFinal: boolean = false) => {
+    if (!wsRef.current || !isConnected) {
+      throw new Error('WebSocket未连接');
+    }
+
+    try {
+      // 直接发送原始PCM数据
+      await wsRef.current.sendRawPCMAudioData(pcmData, isFinal);
+    } catch (error) {
+      console.error('发送原始PCM音频失败:', error);
+      throw error;
+    }
+  }, [isConnected]);
+
   // 开始会话
   const startSession = useCallback(async (language: string = 'zh-CN') => {
     const message: WebSocketMessage = {
@@ -342,6 +394,7 @@ export const useWebSocket = (options: UseWebSocketOptions = {}) => {
     sendMessage,
     sendAudioData,
     sendAudioBlob,
+    sendRawPCMAudio,
     startSession,
     endSession,
     resetChunkCounter,

frontend/src/utils/optimizedAudioRecorder.ts

@@ -178,7 +178,10 @@ export class OptimizedAudioRecorder implements AudioProcessor {
             
             // 实时发送PCM数据
             if (this.onAudioDataCallback) {
+              console.log(`[OptimizedAudioRecorder] 发送实时PCM数据: ${pcmData.length} samples, 功率: ${powerLevel.toFixed(2)}`);
               this.onAudioDataCallback(pcmData);
+            } else {
+              console.warn(`[OptimizedAudioRecorder] 实时音频回调未设置!`);
             }
             
             console.debug(`[OptimizedAudioRecorder] 处理音频块: ${pcmData.length} samples, 功率: ${powerLevel.toFixed(2)}`);

frontend/src/utils/websocketManager.ts

@@ -9,6 +9,7 @@ export class WebSocketService implements WebSocketManager {
   private heartbeatInterval?: number;
   private isReconnecting = false;
   private session: Session | null = null;
+  private isAudioStreamActive = false; // 标记音频流是否活跃
 
   // 事件回调
   private onMessageCallback?: (data: ArrayBuffer | string) => void;
@@ -126,6 +127,13 @@ export class WebSocketService implements WebSocketManager {
       channels: 1
     };
     await this.sendMessage(message);
+    
+    // 重置音频流状态
+    this.isAudioStreamActive = false;
+    console.log("[WebSocketService] 音频流已结束，重置状态");    
+    // 重置音频流状态
+    this.isAudioStreamActive = false;
+    console.log('[WebSocketService] 音频流已结束，重置状态');
     console.log('[WebSocketService] 发送音频开始消息');
   }
 
@@ -136,7 +144,10 @@ export class WebSocketService implements WebSocketManager {
       timestamp: Date.now() / 1000 // 转换为秒级时间戳，与Python保持一致
     };
     await this.sendMessage(message);
-    console.log('[WebSocketService] 发送音频结束消息');
+    
+    // 重置音频流状态
+    this.isAudioStreamActive = false;
+    console.log("[WebSocketService] 音频流已结束，重置状态");    console.log('[WebSocketService] 发送音频结束消息');
   }
 
   async sendAudioData(audioData: ArrayBuffer, chunkId: number, isFinal: boolean = false): Promise<void> {
@@ -197,7 +208,10 @@ export class WebSocketService implements WebSocketManager {
       };
 
       await this.sendMessage(message);
-      console.log('[WebSocketService] PCM音频数据发送成功');
+    
+    // 重置音频流状态
+    this.isAudioStreamActive = false;
+    console.log("[WebSocketService] 音频流已结束，重置状态");      console.log('[WebSocketService] PCM音频数据发送成功');
 
       // 如果是最后一个块，发送音频结束消息
       if (isFinal) {
@@ -248,6 +262,48 @@ export class WebSocketService implements WebSocketManager {
     }
   }
 
+  /**
+   * 直接发送原始PCM音频数据（跳过转换）
+   * @param pcmData Int16Array格式的PCM数据
+   * @param isFinal 是否为最后一个块
+   */
+  async sendRawPCMAudioData(pcmData: Int16Array, isFinal: boolean = false): Promise<void> {
+    try {
+      console.log('[WebSocketService] 开始发送原始PCM数据:', {
+        samples: pcmData.length,
+        size: pcmData.byteLength,
+        format: 'Raw PCM 16kHz 单声道 16bit'
+      });
+
+      // 将Int16Array转换为ArrayBuffer
+      const audioBuffer = pcmData.buffer.slice(pcmData.byteOffset, pcmData.byteOffset + pcmData.byteLength);
+      
+      console.log('[WebSocketService] 原始PCM数据转换完成:', {
+        originalSamples: pcmData.length,
+        bufferSize: audioBuffer.byteLength,
+        format: 'PCM 16kHz 单声道 16bit'
+      });
+
+      // 对于实时音频，我们需要确保发送音频开始消息
+      // 检查是否已经有音频流在进行中
+      if (!this.isAudioStreamActive) {
+        console.log('[WebSocketService] 开始实时音频流，发送音频开始消息');
+        await this.sendAudioStart(this.session?.id || '');
+        this.isAudioStreamActive = true;
+      }
+      
+      // 使用递增的块ID（实时数据不需要重置）
+      const chunkId = Date.now() % 1000000; // 使用时间戳确保唯一性
+      
+      // 发送PCM数据
+      await this.sendAudioData(audioBuffer, chunkId, isFinal);
+      
+    } catch (error) {
+      console.error('[WebSocketService] 发送原始PCM音频失败:', error);
+      throw new Error(`发送原始PCM音频失败: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
   onMessage(callback: (data: ArrayBuffer | string) => void): void {
     this.onMessageCallback = callback;
   }