websocket_handler.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765
  1. """
  2. WebSocket处理器 - 新版本(STT+LLM+TTS流水线,支持流式LLM)
  3. """
  4. import json
  5. import base64
  6. import asyncio
  7. import time
  8. import re
  9. from typing import Dict, Optional
  10. from fastapi import WebSocket
  11. # 预编译正则表达式(性能优化)
  12. EMOTION_TAG_PATTERN = re.compile(r'\[EMOTION:\w+\]')
  13. from models import (
  14. AudioChunkMessage,
  15. AudioEndMessage,
  16. CancelMessage,
  17. TextResultMessage,
  18. EmotionMessage,
  19. AudioResponseMessage,
  20. ErrorMessage,
  21. PingMessage,
  22. PongMessage,
  23. )
  24. from ai_service import ai_service
  25. from request_logger import request_logger
  26. from config import settings
  27. class WebSocketHandler:
  28. """WebSocket连接处理器"""
  29. def __init__(self, websocket: WebSocket):
  30. self.websocket = websocket
  31. self.audio_buffers: Dict[str, bytearray] = {} # requestId -> audio_buffer
  32. self.cancelled_requests: set = set() # 已取消的请求ID
  33. self.recognition_sessions: Dict[str, any] = {} # requestId -> Recognition对象
  34. self.recognition_results: Dict[str, str] = {} # requestId -> 识别结果文本
  35. self.recognition_sentences: Dict[str, list] = {} # requestId -> 句子列表
  36. self.recognition_failed: set = set() # STT识别失败的请求ID
  37. self.event_loop: Optional[asyncio.AbstractEventLoop] = None # 保存事件循环引用
  38. self.request_start_times: Dict[str, float] = {} # 请求开始时间
  39. self.last_audio_frame_time: Dict[str, float] = {} # 最后一帧音频到达时间
  40. self.last_recognition_update_time: Dict[str, float] = {} # 最后一次识别结果更新时间
  41. self.stt_final_events: Dict[str, asyncio.Event] = {} # STT最终结果事件(用于事件驱动)
  42. self.ai_responses: Dict[str, list] = {} # requestId -> AI响应句子列表(用于日志记录)
  43. self.audio_end_times: Dict[str, float] = {} # requestId -> 录音结束时间
  44. async def handle_connection(self):
  45. """处理WebSocket连接"""
  46. await self.websocket.accept()
  47. print("客户端已连接")
  48. # 保存事件循环引用,供回调线程使用
  49. self.event_loop = asyncio.get_running_loop()
  50. try:
  51. while True:
  52. # 接收消息
  53. data = await self.websocket.receive_text()
  54. message_data = json.loads(data)
  55. # 处理消息
  56. await self.handle_message(message_data)
  57. except Exception as e:
  58. print(f"WebSocket错误: {e}")
  59. finally:
  60. print("客户端已断开")
  61. async def handle_message(self, message_data: dict):
  62. """处理接收到的消息"""
  63. message_type = message_data.get("type")
  64. request_id = message_data.get("requestId")
  65. # 检查是否为已取消的请求
  66. if request_id in self.cancelled_requests:
  67. return
  68. if message_type == "audio_chunk":
  69. await self.handle_audio_chunk(AudioChunkMessage(**message_data))
  70. elif message_type == "audio_end":
  71. await self.handle_audio_end(AudioEndMessage(**message_data))
  72. elif message_type == "cancel":
  73. await self.handle_cancel(CancelMessage(**message_data))
  74. elif message_type == "ping":
  75. await self.handle_ping(PingMessage(**message_data))
  76. async def handle_audio_chunk(self, message: AudioChunkMessage):
  77. """处理音频数据块 - 流式STT"""
  78. request_id = message.requestId
  79. # 检查是否取消或STT已失败
  80. if request_id in self.cancelled_requests or request_id in self.recognition_failed:
  81. return
  82. # Base64解码音频数据
  83. audio_data = base64.b64decode(message.data)
  84. chunk_size = len(audio_data)
  85. # 初始化buffer(用于备份,以防STT失败)
  86. if request_id not in self.audio_buffers:
  87. self.audio_buffers[request_id] = bytearray()
  88. self.request_start_times[request_id] = time.time()
  89. print(f"\n[{request_id}] 🎤 开始接收音频流 [时间: {time.strftime('%H:%M:%S.%f')[:-3]}]")
  90. print(f"[{request_id}] 音频参数: {message.sampleRate}Hz, {message.channels}ch, {message.bitDepth}bit")
  91. self.audio_buffers[request_id].extend(audio_data)
  92. total_size = len(self.audio_buffers[request_id])
  93. # 计算已录制时长(16000 sample_rate, 16bit = 2 bytes per sample, mono)
  94. duration_seconds = total_size / (16000 * 2)
  95. # 打印音频块信息
  96. print(f"[{request_id}] 📦 接收chunk#{message.chunkIndex}: "
  97. f"大小={chunk_size}字节, 累计={total_size}字节 ({duration_seconds:.2f}秒)")
  98. # 第一个chunk:启动流式STT识别
  99. if request_id not in self.recognition_sessions:
  100. print(f"[{request_id}] 🚀 首个chunk,启动STT流式识别...")
  101. success = await self.start_streaming_stt(request_id)
  102. if not success:
  103. # STT启动失败,标记为失败状态
  104. print(f"[{request_id}] ❌ STT启动失败")
  105. self.recognition_failed.add(request_id)
  106. return
  107. # 🔍 记录最后一帧音频到达时间
  108. self.last_audio_frame_time[request_id] = time.time()
  109. # 发送音频帧到STT服务
  110. recognition = self.recognition_sessions.get(request_id)
  111. if recognition:
  112. try:
  113. # 在线程池中发送音频帧
  114. send_start = time.time()
  115. await asyncio.to_thread(recognition.send_audio_frame, audio_data)
  116. send_duration = (time.time() - send_start) * 1000
  117. print(f"[{request_id}] ➡️ 发送至STT: {chunk_size}字节 (耗时: {send_duration:.1f}ms)")
  118. except Exception as e:
  119. # 发送失败,标记为失败状态,避免后续继续发送
  120. if request_id not in self.recognition_failed:
  121. print(f"[{request_id}] ❌ 发送音频帧错误,停止发送: {e}")
  122. self.recognition_failed.add(request_id)
  123. async def handle_audio_end(self, message: AudioEndMessage):
  124. """处理音频结束 - 使用流式LLM优化"""
  125. request_id = message.requestId
  126. # 记录录音结束时间
  127. audio_end_time = time.time()
  128. self.audio_end_times[request_id] = audio_end_time
  129. # 初始化AI响应收集器
  130. self.ai_responses[request_id] = []
  131. # 统计音频流信息
  132. total_size = len(self.audio_buffers.get(request_id, bytearray()))
  133. duration_seconds = total_size / (16000 * 2)
  134. total_chunks = message.totalChunks
  135. print(f"\n[{request_id}] 🎤 音频录制结束 [时间: {time.strftime('%H:%M:%S.%f')[:-3]}]")
  136. print(f"[{request_id}] 📊 音频流统计: 总大小={total_size}字节, 时长={duration_seconds:.2f}秒, 共{total_chunks}个chunk")
  137. # 🔍 时间分析:最后一帧音频到达时间
  138. last_frame_time = self.last_audio_frame_time.get(request_id, 0)
  139. if last_frame_time:
  140. frame_delay = (audio_end_time - last_frame_time) * 1000
  141. print(f"[{request_id}] 🔍 最后音频帧到达 → audio_end消息: {frame_delay:.1f}ms")
  142. # 检查是否取消
  143. if request_id in self.cancelled_requests:
  144. print(f"[{request_id}] 请求已取消")
  145. self.cancelled_requests.remove(request_id)
  146. await self.cleanup_request(request_id)
  147. return
  148. try:
  149. # 1. STT: 等待完整识别结果(保证准确性)
  150. stt_start = time.time()
  151. # 🔥 触发STT异步停止
  152. stop_trigger_time = time.time()
  153. await self.stop_streaming_stt(request_id)
  154. stop_return_time = time.time()
  155. stop_blocking = (stop_return_time - stop_trigger_time) * 1000
  156. print(f"[{request_id}] ⚡ STT stop()返回 (阻塞时间: {stop_blocking:.1f}ms) [目标: <10ms]")
  157. # 🔥 优化:并行预热LLM连接(在等待STT期间)
  158. async def warmup_llm_connection():
  159. """预热LLM HTTP连接,减少首次请求延迟"""
  160. try:
  161. print(f"[{request_id}] 🔥 预热LLM连接...")
  162. # 发送极短请求以建立连接(不等待完整响应)
  163. await asyncio.wait_for(
  164. asyncio.to_thread(
  165. lambda: ai_service.openai_client.chat.completions.create(
  166. model=settings.llm_model,
  167. messages=[{"role": "user", "content": "."}],
  168. max_tokens=1,
  169. stream=False,
  170. )
  171. ),
  172. timeout=0.5 # 最多等待500ms
  173. )
  174. print(f"[{request_id}] ✅ LLM连接预热成功")
  175. except Exception as e:
  176. # 预热失败不影响主流程
  177. print(f"[{request_id}] ⚠️ LLM预热失败(忽略): {e}")
  178. # 🔥 优化:动态超时等待 - 根据最后更新时间智能调整
  179. print(f"[{request_id}] ⏳ 等待STT最终识别结果(动态超时)+ 预热LLM...")
  180. wait_start = time.time()
  181. # 🔍 检查最后一次识别结果更新时间,动态调整超时
  182. last_update = self.last_recognition_update_time.get(request_id, 0)
  183. if last_update:
  184. time_since_update = (wait_start - last_update) * 1000
  185. print(f"[{request_id}] 🔍 当前时刻距最后一次识别更新: {time_since_update:.1f}ms")
  186. # 🎯 动态超时策略:根据最近更新时间调整
  187. if time_since_update < 50:
  188. # 刚更新过,使用短超时
  189. timeout = 0.1
  190. print(f"[{request_id}] 📊 动态超时: 100ms(最近刚更新)")
  191. elif time_since_update < 150:
  192. # 中等时间,使用中等超时
  193. timeout = 0.2
  194. print(f"[{request_id}] 📊 动态超时: 200ms(中等间隔)")
  195. else:
  196. # 较长时间,使用标准超时
  197. timeout = 0.3
  198. print(f"[{request_id}] 📊 动态超时: 300ms(较长间隔)")
  199. else:
  200. # 没有更新记录,使用保守超时
  201. timeout = 0.3
  202. print(f"[{request_id}] 📊 动态超时: 300ms(无更新记录)")
  203. # 🔥 并行执行:等待STT + 预热LLM(节省时间)
  204. final_event = self.stt_final_events.get(request_id)
  205. # 创建等待任务
  206. if final_event:
  207. stt_wait_task = asyncio.create_task(
  208. asyncio.wait_for(final_event.wait(), timeout=timeout)
  209. )
  210. else:
  211. # 降级到固定等待
  212. stt_wait_task = asyncio.create_task(asyncio.sleep(timeout))
  213. # 创建预热任务
  214. warmup_task = asyncio.create_task(warmup_llm_connection())
  215. # 并行执行(谁先完成算谁,但都不阻塞)
  216. got_final_event = False
  217. try:
  218. # 等待STT完成,同时预热在后台进行
  219. await stt_wait_task
  220. print(f"[{request_id}] ⚡ 收到STT最终结果事件!")
  221. got_final_event = True
  222. except asyncio.TimeoutError:
  223. print(f"[{request_id}] ⏰ 等待超时({int(timeout*1000)}ms),再等待50ms给最终结果一个机会...")
  224. # 🔧 关键修复:超时后再等待一小段时间,给STT最终结果到达的机会
  225. # 因为STT的最终结果可能在事件触发失败的情况下延迟到达
  226. try:
  227. if final_event:
  228. await asyncio.wait_for(final_event.wait(), timeout=0.05) # 额外等待50ms
  229. print(f"[{request_id}] ✅ 额外等待期间收到最终结果!")
  230. got_final_event = True
  231. except asyncio.TimeoutError:
  232. print(f"[{request_id}] ⏰ 额外等待也超时,使用当前结果")
  233. # 确保预热任务也完成(或超时)
  234. try:
  235. await asyncio.wait_for(warmup_task, timeout=0.1)
  236. except:
  237. pass # 预热失败不影响
  238. wait_actual = (time.time() - wait_start) * 1000
  239. print(f"[{request_id}] ✓ STT等待完成 (实际: {wait_actual:.1f}ms)")
  240. # 🔍 再次检查识别结果更新时间
  241. last_update_after = self.last_recognition_update_time.get(request_id, 0)
  242. if last_update_after and last_update:
  243. if last_update_after > last_update:
  244. new_update_delay = (last_update_after - wait_start) * 1000
  245. print(f"[{request_id}] 🔍 ✅ 等待期间有新的识别结果更新!延迟: {new_update_delay:.1f}ms")
  246. else:
  247. print(f"[{request_id}] 🔍 ⚠️ 等待期间没有新的识别结果更新")
  248. # 🔧 关键修复:如果没有收到最终结果事件,再主动等待一小段时间让 recognition_results 更新
  249. if not got_final_event:
  250. print(f"[{request_id}] 🔄 未收到最终事件,再等待100ms让识别结果更新...")
  251. await asyncio.sleep(0.1)
  252. # 再次检查更新
  253. last_update_final = self.last_recognition_update_time.get(request_id, 0)
  254. if last_update_final > last_update_after:
  255. print(f"[{request_id}] ✅ 最终等待成功!识别结果已更新")
  256. else:
  257. print(f"[{request_id}] ⚠️ 最终等待后仍无更新,使用现有结果")
  258. # 获取STT识别结果
  259. text = self.recognition_results.get(request_id, "")
  260. text_length = len(text) if text else 0
  261. if not text:
  262. print(f"[{request_id}] ⚠️ STT未识别到文本,使用默认")
  263. text = "Hello, how can I help you?"
  264. else:
  265. print(f"[{request_id}] ✓ STT识别完成: \"{text}\" (长度: {text_length}字)")
  266. # 发送最终识别结果
  267. await self.send_text_result(request_id, text, True)
  268. stt_end = time.time()
  269. stt_duration = stt_end - self.request_start_times.get(request_id, stt_start)
  270. print(f"[{request_id}] ⏱️ STT总耗时: {stt_duration:.2f}秒")
  271. if request_id in self.cancelled_requests:
  272. return
  273. # 2+3. LLM流式 + TTS并行(优化:边生成边播放)
  274. print(f"[{request_id}] 步骤 2+3: LLM流式 + TTS并行")
  275. llm_start = time.time()
  276. chunk_index = 0
  277. first_audio_time = None
  278. final_emotion = "neutral"
  279. sentence_count = 0
  280. llm_first_sentence_time = None
  281. emotion_sent = False # 标记是否已发送情感
  282. # 🔥 流式处理:LLM生成一句 → 立即TTS
  283. async for sentence, emotion in ai_service.generate_response_stream(text):
  284. if request_id in self.cancelled_requests:
  285. return
  286. # 🎯 优先处理情感:第一时间发送给前端
  287. if not emotion_sent and emotion:
  288. await self.send_emotion(request_id, emotion)
  289. print(f"[{request_id}] 🎭 立即发送情感给前端: {emotion}")
  290. emotion_sent = True
  291. final_emotion = emotion
  292. # ⚡ 如果是空句子(仅传递情感),跳过 TTS 处理
  293. if not sentence:
  294. print(f"[{request_id}] ⚡ 收到仅情感标记(无句子内容),继续等待实际文本")
  295. continue
  296. sentence_count += 1
  297. # 🧹 清理文本:移除情感标记和特殊字符(保留文字内容)
  298. clean_sentence = EMOTION_TAG_PATTERN.sub('', sentence)
  299. # 移除括号符号本身,但保留括号内的文字(避免TTS发音异常)
  300. # 使用 str.translate() 比多次 replace() 更快
  301. clean_sentence = clean_sentence.translate(str.maketrans('', '', '()[]')).strip()
  302. # 如果清理后为空,跳过
  303. if not clean_sentence:
  304. print(f"[{request_id}] ⚠️ 跳过空句子(仅含情感标记)")
  305. continue
  306. # 收集AI响应(用于日志记录)
  307. self.ai_responses[request_id].append(clean_sentence)
  308. # 记录第一个句子生成时间
  309. if llm_first_sentence_time is None:
  310. llm_first_sentence_time = time.time()
  311. llm_first_sentence_duration = llm_first_sentence_time - llm_start
  312. print(f"[{request_id}] 💬 LLM第{sentence_count}句: {clean_sentence} (耗时: {llm_first_sentence_duration:.2f}秒)")
  313. else:
  314. print(f"[{request_id}] 💬 LLM第{sentence_count}句: {clean_sentence}")
  315. print(f"[{request_id}] 🎵 立即启动TTS合成第{sentence_count}句...")
  316. tts_sentence_start = time.time()
  317. # 立即TTS合成这个句子(使用清理后的文本 + 情感参数)
  318. async for audio_chunk in ai_service.text_to_speech_stream(clean_sentence, emotion):
  319. if request_id in self.cancelled_requests:
  320. return
  321. # Base64编码
  322. audio_base64 = base64.b64encode(audio_chunk).decode()
  323. # 发送音频数据
  324. await self.send_audio_response(
  325. request_id,
  326. audio_base64,
  327. chunk_index,
  328. False,
  329. )
  330. # 记录第一个音频块发送时间
  331. if chunk_index == 0:
  332. first_audio_time = time.time()
  333. time_to_first_audio = first_audio_time - audio_end_time
  334. print(f"[{request_id}] 🎵 第一个音频块已发送 [耗时: {time_to_first_audio:.2f}秒 = 录音结束→第一个音频块] ⚡")
  335. chunk_index += 1
  336. tts_sentence_duration = time.time() - tts_sentence_start
  337. print(f"[{request_id}] ✓ 第{sentence_count}句TTS完成 (耗时: {tts_sentence_duration:.2f}秒)")
  338. # 更新当前情感(但不再发送,因为已在开头发送)
  339. if emotion:
  340. final_emotion = emotion
  341. # 发送TTS结束标记
  342. await self.send_audio_response(request_id, "", chunk_index, True)
  343. # 记录最后一个音频块发送时间
  344. last_audio_time = time.time()
  345. time_to_last_audio = last_audio_time - audio_end_time
  346. print(f"[{request_id}] 🎵 最后一个音频块已发送 [耗时: {time_to_last_audio:.2f}秒 = 录音结束→最后一个音频块]")
  347. # 计算总耗时
  348. total_end = time.time()
  349. total_duration = total_end - self.request_start_times.get(request_id, stt_start)
  350. llm_tts_total = total_end - llm_start
  351. # 性能汇总日志
  352. print(f"\n{'='*60}")
  353. print(f"[{request_id}] 📊 性能汇总报告(流式优化)")
  354. print(f"{'='*60}")
  355. print(f" STT 耗时: {stt_duration:.2f}秒")
  356. print(f" LLM+TTS 流式总耗时: {llm_tts_total:.2f}秒")
  357. print(f" ├─ LLM第1句: {llm_first_sentence_duration:.2f}秒" if llm_first_sentence_time else "")
  358. print(f" └─ 共{sentence_count}个句子")
  359. if first_audio_time:
  360. print(f" 🎯 录音结束→首个音频: {time_to_first_audio:.2f}秒 ⭐ [流式优化]")
  361. print(f" 🎯 录音结束→全部音频: {time_to_last_audio:.2f}秒")
  362. print(f" 总耗时: {total_duration:.2f}秒")
  363. print(f"{'='*60}")
  364. print(f" 📝 优化验证指标:")
  365. print(f" - STT等待实际耗时: {wait_actual:.0f}ms (期望: ~300ms, 前端100ms发送间隔)")
  366. print(f" - STT stop阻塞时间: {stop_blocking:.0f}ms (期望: <10ms)")
  367. print(f" - 识别文本长度: {text_length}字 (检查完整性)")
  368. if first_audio_time:
  369. baseline_time = 1.57 # 串行基准:LLM(0.77s) + TTS(0.8s)
  370. improvement = baseline_time - time_to_first_audio
  371. improvement_pct = (improvement / baseline_time) * 100
  372. print(f" - 流式优化效果: {improvement:+.2f}秒 ({improvement_pct:+.1f}%) [基准: {baseline_time}s]")
  373. print(f" - 用户感知延迟: {time_to_first_audio:.2f}秒 [目标: <0.5s]")
  374. print(f"{'='*60}")
  375. print(f"[{request_id}] ✅ 请求处理完成\n")
  376. # 📝 记录对话日志到文件
  377. try:
  378. user_text_log = text
  379. ai_response_log = ''.join(self.ai_responses.get(request_id, []))
  380. first_audio_latency_ms = (first_audio_time - audio_end_time) * 1000 if first_audio_time else None
  381. request_logger.log_request(
  382. user_text=user_text_log,
  383. ai_response=ai_response_log,
  384. first_audio_latency=first_audio_latency_ms,
  385. request_id=request_id
  386. )
  387. except Exception as log_error:
  388. print(f"[{request_id}] ⚠️ 记录日志失败: {log_error}")
  389. except Exception as e:
  390. print(f"[{request_id}] 处理错误: {e}")
  391. import traceback
  392. traceback.print_exc()
  393. # 记录错误日志
  394. try:
  395. user_text_log = self.recognition_results.get(request_id, "")
  396. request_logger.log_error(
  397. user_text=user_text_log,
  398. error_message=str(e),
  399. request_id=request_id
  400. )
  401. except:
  402. pass
  403. await self.send_error(request_id, "NETWORK_ERROR", str(e))
  404. finally:
  405. # 清理资源
  406. await self.cleanup_request(request_id)
  407. async def start_streaming_stt(self, request_id: str) -> bool:
  408. """启动流式STT识别,返回是否成功"""
  409. try:
  410. stt_init_start = time.time()
  411. print(f"[{request_id}] 🔊 启动流式STT识别...")
  412. # 发送"正在听..."的中间状态
  413. await self.send_text_result(request_id, "Listening...", False)
  414. from dashscope.audio.asr import Recognition, RecognitionCallback, RecognitionResult
  415. # 定义回调处理类
  416. handler = self
  417. req_id = request_id
  418. class StreamCallback(RecognitionCallback):
  419. def on_open(self):
  420. print(f"[{req_id}] 🔗 STT连接已打开")
  421. def on_close(self):
  422. print(f"[{req_id}] 🔌 STT连接已关闭")
  423. # 标记为失败,停止后续音频帧发送
  424. handler.recognition_failed.add(req_id)
  425. def on_error(self, error):
  426. print(f"[{req_id}] ❌ STT错误: {error}")
  427. # 标记为失败
  428. handler.recognition_failed.add(req_id)
  429. def on_event(self, result: RecognitionResult):
  430. # 获取识别的句子
  431. sentence = result.get_sentence()
  432. if sentence and 'text' in sentence:
  433. text = sentence['text']
  434. begin_time = sentence.get('begin_time', 0)
  435. end_time = sentence.get('end_time')
  436. is_final = end_time is not None and end_time > 0
  437. status = "✅最终" if is_final else "⏳中间"
  438. print(f"[{req_id}] 💬 STT识别({status}): {text} [时间: {begin_time}-{end_time}]")
  439. # 初始化句子列表
  440. if req_id not in handler.recognition_sentences:
  441. handler.recognition_sentences[req_id] = []
  442. sentence_list = handler.recognition_sentences[req_id]
  443. # 查找是否是已有句子的更新(根据begin_time匹配)
  444. updated = False
  445. for i, sent in enumerate(sentence_list):
  446. if abs(sent['begin_time'] - begin_time) < 100: # 100ms容差
  447. # 更新现有句子
  448. sentence_list[i] = {'text': text, 'begin_time': begin_time, 'end_time': end_time}
  449. updated = True
  450. break
  451. if not updated:
  452. # 新句子,追加
  453. sentence_list.append({'text': text, 'begin_time': begin_time, 'end_time': end_time})
  454. # 按时间排序并合并所有句子
  455. sentence_list.sort(key=lambda x: x['begin_time'])
  456. accumulated_text = ' '.join([s['text'] for s in sentence_list])
  457. # 🔍 记录识别结果更新时间
  458. handler.last_recognition_update_time[req_id] = time.time()
  459. handler.recognition_results[req_id] = accumulated_text
  460. print(f"[{req_id}] 📝 累积文本: {accumulated_text} [更新时间: {time.strftime('%H:%M:%S.%f')[:-3]}]")
  461. # 🔥 如果是最终结果,触发事件
  462. if is_final and req_id in handler.stt_final_events:
  463. try:
  464. handler.event_loop.call_soon_threadsafe(handler.stt_final_events[req_id].set)
  465. print(f"[{req_id}] ⚡ 触发STT最终结果事件")
  466. except Exception as e:
  467. print(f"[{req_id}] ❌ 触发事件失败: {e}")
  468. # 从回调线程安全地发送到事件循环
  469. if handler.event_loop:
  470. try:
  471. asyncio.run_coroutine_threadsafe(
  472. handler.send_text_result(req_id, accumulated_text, False),
  473. handler.event_loop
  474. )
  475. except Exception as e:
  476. print(f"[{req_id}] ❌ 发送中间结果失败: {e}")
  477. # 创建识别对象
  478. def create_recognition():
  479. return Recognition(
  480. model='paraformer-realtime-v2',
  481. format='pcm',
  482. sample_rate=16000,
  483. callback=StreamCallback()
  484. )
  485. recognition = await asyncio.to_thread(create_recognition)
  486. # 启动识别
  487. await asyncio.to_thread(recognition.start)
  488. # 保存识别会话
  489. self.recognition_sessions[request_id] = recognition
  490. # 创建最终结果事件
  491. self.stt_final_events[request_id] = asyncio.Event()
  492. stt_init_duration = (time.time() - stt_init_start) * 1000
  493. print(f"[{request_id}] ✅ STT识别会话已启动 (耗时: {stt_init_duration:.1f}ms)")
  494. # 🔥 优化:移除不必要的100ms延迟,直接开始
  495. # await asyncio.sleep(0.1) # 已移除
  496. # 检查是否已经失败
  497. if request_id in self.recognition_failed:
  498. print(f"[{request_id}] ❌ STT连接启动后立即失败")
  499. return False
  500. return True
  501. except Exception as e:
  502. print(f"[{request_id}] 启动STT识别失败: {e}")
  503. import traceback
  504. traceback.print_exc()
  505. return False
  506. async def stop_streaming_stt(self, request_id: str):
  507. """停止流式STT识别(异步非阻塞)"""
  508. try:
  509. recognition = self.recognition_sessions.get(request_id)
  510. if recognition:
  511. print(f"[{request_id}] 🛑 触发STT异步停止...")
  512. # 🔥 优化:异步停止STT,不等待完成
  513. # 这样可以立即进入LLM处理,节省200-340ms
  514. async def _async_stop():
  515. try:
  516. stop_start = time.time()
  517. await asyncio.to_thread(recognition.stop)
  518. stop_duration = (time.time() - stop_start) * 1000
  519. print(f"[{request_id}] ✅ STT识别已停止 (耗时: {stop_duration:.1f}ms)")
  520. except Exception as e:
  521. print(f"[{request_id}] ⚠️ STT停止异常: {e}")
  522. # 启动异步任务但不等待
  523. asyncio.create_task(_async_stop())
  524. print(f"[{request_id}] ⚡ STT异步停止已触发,继续处理...")
  525. except Exception as e:
  526. print(f"[{request_id}] ❌ 停止STT识别错误: {e}")
  527. async def cleanup_request(self, request_id: str):
  528. """清理请求相关资源"""
  529. # 清理audio buffer
  530. if request_id in self.audio_buffers:
  531. del self.audio_buffers[request_id]
  532. # 清理识别会话
  533. if request_id in self.recognition_sessions:
  534. del self.recognition_sessions[request_id]
  535. # 清理识别结果
  536. if request_id in self.recognition_results:
  537. del self.recognition_results[request_id]
  538. # 清理句子列表
  539. if request_id in self.recognition_sentences:
  540. del self.recognition_sentences[request_id]
  541. # 清理取消标记
  542. if request_id in self.cancelled_requests:
  543. self.cancelled_requests.remove(request_id)
  544. # 清理失败标记
  545. if request_id in self.recognition_failed:
  546. self.recognition_failed.remove(request_id)
  547. # 清理时间记录
  548. if request_id in self.request_start_times:
  549. del self.request_start_times[request_id]
  550. if request_id in self.last_audio_frame_time:
  551. del self.last_audio_frame_time[request_id]
  552. if request_id in self.last_recognition_update_time:
  553. del self.last_recognition_update_time[request_id]
  554. if request_id in self.audio_end_times:
  555. del self.audio_end_times[request_id]
  556. # 清理AI响应收集器
  557. if request_id in self.ai_responses:
  558. del self.ai_responses[request_id]
  559. # 清理事件
  560. if request_id in self.stt_final_events:
  561. del self.stt_final_events[request_id]
  562. print(f"[{request_id}] 资源已清理")
  563. async def handle_cancel(self, message: CancelMessage):
  564. """处理取消请求"""
  565. request_id = message.requestId
  566. print(f"[{request_id}] 收到取消请求")
  567. self.cancelled_requests.add(request_id)
  568. # 停止STT识别
  569. await self.stop_streaming_stt(request_id)
  570. # 清理资源
  571. await self.cleanup_request(request_id)
  572. async def handle_ping(self, message: PingMessage):
  573. """处理Ping"""
  574. await self.send_pong(message.requestId)
  575. async def send_text_result(self, request_id: str, text: str, is_final: bool):
  576. """发送文本结果"""
  577. message = TextResultMessage(
  578. type="text_result",
  579. requestId=request_id,
  580. timestamp=self._get_timestamp(),
  581. text=text,
  582. isFinal=is_final,
  583. )
  584. await self.websocket.send_text(message.model_dump_json())
  585. async def send_emotion(self, request_id: str, emotion: str):
  586. """发送情感标签"""
  587. message = EmotionMessage(
  588. type="emotion",
  589. requestId=request_id,
  590. timestamp=self._get_timestamp(),
  591. emotion=emotion,
  592. )
  593. await self.websocket.send_text(message.model_dump_json())
  594. async def send_audio_response(
  595. self, request_id: str, data: str, chunk_index: int, is_last: bool,
  596. format: str = "pcm", sample_rate: int = 24000
  597. ):
  598. """发送音频响应"""
  599. message = AudioResponseMessage(
  600. type="audio_response",
  601. requestId=request_id,
  602. timestamp=self._get_timestamp(),
  603. data=data,
  604. sampleRate=sample_rate,
  605. format=format,
  606. chunkIndex=chunk_index,
  607. isLast=is_last,
  608. )
  609. await self.websocket.send_text(message.model_dump_json())
  610. async def send_error(self, request_id: str, code: str, error_message: str):
  611. """发送错误消息"""
  612. message = ErrorMessage(
  613. type="error",
  614. requestId=request_id,
  615. timestamp=self._get_timestamp(),
  616. code=code,
  617. message=error_message,
  618. )
  619. await self.websocket.send_text(message.model_dump_json())
  620. async def send_pong(self, request_id: str):
  621. """发送Pong"""
  622. message = PongMessage(
  623. type="pong",
  624. requestId=request_id,
  625. timestamp=self._get_timestamp(),
  626. )
  627. await self.websocket.send_text(message.model_dump_json())
  628. def _get_timestamp(self) -> int:
  629. """获取当前时间戳(毫秒)"""
  630. return int(time.time() * 1000)