websocket_handler.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833
  1. """
  2. WebSocket处理器 - 新版本(STT+LLM+TTS流水线,支持流式LLM)
  3. """
  4. import json
  5. import base64
  6. import asyncio
  7. import time
  8. import re
  9. from typing import Dict, Optional
  10. from fastapi import WebSocket
  11. # 预编译正则表达式(性能优化)
  12. EMOTION_TAG_PATTERN = re.compile(r'\[EMOTION:\w+\]')
  13. from models import (
  14. AudioChunkMessage,
  15. AudioEndMessage,
  16. CancelMessage,
  17. TextResultMessage,
  18. EmotionMessage,
  19. AudioResponseMessage,
  20. ErrorMessage,
  21. PingMessage,
  22. PongMessage,
  23. )
  24. from ai_service import ai_service
  25. from request_logger import request_logger
  26. from config import settings, get_dashscope_ws_url
  27. class WebSocketHandler:
  28. """WebSocket连接处理器"""
  29. def __init__(self, websocket: WebSocket):
  30. self.websocket = websocket
  31. self.audio_buffers: Dict[str, bytearray] = {} # requestId -> audio_buffer
  32. self.cancelled_requests: set = set() # 已取消的请求ID
  33. self.recognition_sessions: Dict[str, any] = {} # requestId -> OmniRealtimeConversation对象
  34. self.recognition_results: Dict[str, str] = {} # requestId -> 识别结果文本
  35. self.recognition_failed: set = set() # STT识别失败的请求ID
  36. self.event_loop: Optional[asyncio.AbstractEventLoop] = None # 保存事件循环引用
  37. self.request_start_times: Dict[str, float] = {} # 请求开始时间
  38. self.last_audio_frame_time: Dict[str, float] = {} # 最后一帧音频到达时间
  39. self.last_recognition_update_time: Dict[str, float] = {} # 最后一次识别结果更新时间
  40. self.stt_final_events: Dict[str, asyncio.Event] = {} # STT最终结果事件(用于事件驱动)
  41. self.ai_responses: Dict[str, list] = {} # requestId -> AI响应句子列表(用于日志记录)
  42. self.audio_end_times: Dict[str, float] = {} # requestId -> 录音结束时间
  43. async def handle_connection(self):
  44. """处理WebSocket连接"""
  45. await self.websocket.accept()
  46. print("客户端已连接")
  47. # 保存事件循环引用,供回调线程使用
  48. self.event_loop = asyncio.get_running_loop()
  49. try:
  50. while True:
  51. # 接收消息
  52. data = await self.websocket.receive_text()
  53. message_data = json.loads(data)
  54. # 处理消息
  55. await self.handle_message(message_data)
  56. except Exception as e:
  57. print(f"WebSocket错误: {e}")
  58. finally:
  59. print("客户端已断开")
  60. async def handle_message(self, message_data: dict):
  61. """处理接收到的消息"""
  62. message_type = message_data.get("type")
  63. request_id = message_data.get("requestId")
  64. # 检查是否为已取消的请求
  65. if request_id in self.cancelled_requests:
  66. return
  67. if message_type == "audio_chunk":
  68. await self.handle_audio_chunk(AudioChunkMessage(**message_data))
  69. elif message_type == "audio_end":
  70. await self.handle_audio_end(AudioEndMessage(**message_data))
  71. elif message_type == "cancel":
  72. await self.handle_cancel(CancelMessage(**message_data))
  73. elif message_type == "ping":
  74. await self.handle_ping(PingMessage(**message_data))
  75. async def handle_audio_chunk(self, message: AudioChunkMessage):
  76. """处理音频数据块 - 流式STT"""
  77. request_id = message.requestId
  78. # 检查是否取消或STT已失败
  79. if request_id in self.cancelled_requests or request_id in self.recognition_failed:
  80. return
  81. # Base64解码音频数据
  82. audio_data = base64.b64decode(message.data)
  83. chunk_size = len(audio_data)
  84. # 初始化buffer(用于备份,以防STT失败)
  85. if request_id not in self.audio_buffers:
  86. self.audio_buffers[request_id] = bytearray()
  87. self.request_start_times[request_id] = time.time()
  88. print(f"\n[{request_id}] 🎤 开始接收音频流 [时间: {time.strftime('%H:%M:%S.%f')[:-3]}]")
  89. print(f"[{request_id}] 音频参数: {message.sampleRate}Hz, {message.channels}ch, {message.bitDepth}bit")
  90. self.audio_buffers[request_id].extend(audio_data)
  91. total_size = len(self.audio_buffers[request_id])
  92. # 计算已录制时长(16000 sample_rate, 16bit = 2 bytes per sample, mono)
  93. duration_seconds = total_size / (16000 * 2)
  94. # 打印音频块信息
  95. print(f"[{request_id}] 📦 接收chunk#{message.chunkIndex}: "
  96. f"大小={chunk_size}字节, 累计={total_size}字节 ({duration_seconds:.2f}秒)")
  97. # 第一个chunk:启动流式STT识别
  98. if request_id not in self.recognition_sessions:
  99. print(f"[{request_id}] 🚀 首个chunk,启动STT流式识别...")
  100. success = await self.start_streaming_stt(request_id)
  101. if not success:
  102. # STT启动失败,标记为失败状态
  103. print(f"[{request_id}] ❌ STT启动失败")
  104. self.recognition_failed.add(request_id)
  105. return
  106. # 🔍 记录最后一帧音频到达时间
  107. self.last_audio_frame_time[request_id] = time.time()
  108. # 发送音频帧到STT服务(OmniRealtimeConversation使用base64编码)
  109. conversation = self.recognition_sessions.get(request_id)
  110. if conversation:
  111. try:
  112. # Base64编码音频数据
  113. audio_b64 = base64.b64encode(audio_data).decode('ascii')
  114. # 在线程池中发送音频帧
  115. send_start = time.time()
  116. await asyncio.to_thread(conversation.append_audio, audio_b64)
  117. send_duration = (time.time() - send_start) * 1000
  118. print(f"[{request_id}] ➡️ 发送至STT: {chunk_size}字节 (耗时: {send_duration:.1f}ms)")
  119. except Exception as e:
  120. # 发送失败,标记为失败状态,避免后续继续发送
  121. if request_id not in self.recognition_failed:
  122. print(f"[{request_id}] ❌ 发送音频帧错误,停止发送: {e}")
  123. self.recognition_failed.add(request_id)
  124. async def handle_audio_end(self, message: AudioEndMessage):
  125. """处理音频结束 - 使用流式LLM优化"""
  126. request_id = message.requestId
  127. # 记录录音结束时间
  128. audio_end_time = time.time()
  129. self.audio_end_times[request_id] = audio_end_time
  130. # 初始化AI响应收集器
  131. self.ai_responses[request_id] = []
  132. # 统计音频流信息
  133. total_size = len(self.audio_buffers.get(request_id, bytearray()))
  134. duration_seconds = total_size / (16000 * 2)
  135. total_chunks = message.totalChunks
  136. print(f"\n[{request_id}] 🎤 音频录制结束 [时间: {time.strftime('%H:%M:%S.%f')[:-3]}]")
  137. print(f"[{request_id}] 📊 音频流统计: 总大小={total_size}字节, 时长={duration_seconds:.2f}秒, 共{total_chunks}个chunk")
  138. # 🔍 时间分析:最后一帧音频到达时间
  139. last_frame_time = self.last_audio_frame_time.get(request_id, 0)
  140. if last_frame_time:
  141. frame_delay = (audio_end_time - last_frame_time) * 1000
  142. print(f"[{request_id}] 🔍 最后音频帧到达 → audio_end消息: {frame_delay:.1f}ms")
  143. # 检查是否取消
  144. if request_id in self.cancelled_requests:
  145. print(f"[{request_id}] 请求已取消")
  146. self.cancelled_requests.remove(request_id)
  147. await self.cleanup_request(request_id)
  148. return
  149. try:
  150. # 1. STT: 等待完整识别结果(保证准确性)
  151. stt_start = time.time()
  152. # 🔥 触发STT异步停止
  153. stop_trigger_time = time.time()
  154. await self.stop_streaming_stt(request_id)
  155. stop_return_time = time.time()
  156. stop_blocking = (stop_return_time - stop_trigger_time) * 1000
  157. print(f"[{request_id}] ⚡ STT stop()返回 (阻塞时间: {stop_blocking:.1f}ms) [目标: <10ms]")
  158. # 🔥 优化:并行预热LLM连接(在等待STT期间)
  159. async def warmup_llm_connection():
  160. """预热LLM HTTP连接,减少首次请求延迟"""
  161. try:
  162. print(f"[{request_id}] 🔥 预热LLM连接...")
  163. # 发送极短请求以建立连接(不等待完整响应)
  164. await asyncio.wait_for(
  165. asyncio.to_thread(
  166. lambda: ai_service.openai_client.chat.completions.create(
  167. model=settings.llm_model,
  168. messages=[{"role": "user", "content": "."}],
  169. max_tokens=1,
  170. stream=False,
  171. )
  172. ),
  173. timeout=0.5 # 最多等待500ms
  174. )
  175. print(f"[{request_id}] ✅ LLM连接预热成功")
  176. except Exception as e:
  177. # 预热失败不影响主流程
  178. print(f"[{request_id}] ⚠️ LLM预热失败(忽略): {e}")
  179. # 🔥 优化:动态超时等待 - 根据最后更新时间智能调整
  180. print(f"[{request_id}] ⏳ 等待STT最终识别结果(动态超时)+ 预热LLM...")
  181. wait_start = time.time()
  182. # 🔍 检查最后一次识别结果更新时间,动态调整超时
  183. last_update = self.last_recognition_update_time.get(request_id, 0)
  184. if last_update:
  185. time_since_update = (wait_start - last_update) * 1000
  186. print(f"[{request_id}] 🔍 当前时刻距最后一次识别更新: {time_since_update:.1f}ms")
  187. # 🎯 动态超时策略:根据最近更新时间调整
  188. if time_since_update < 50:
  189. # 刚更新过,使用短超时
  190. timeout = 0.1
  191. print(f"[{request_id}] 📊 动态超时: 100ms(最近刚更新)")
  192. elif time_since_update < 150:
  193. # 中等时间,使用中等超时
  194. timeout = 0.2
  195. print(f"[{request_id}] 📊 动态超时: 200ms(中等间隔)")
  196. else:
  197. # 较长时间,使用标准超时
  198. timeout = 0.3
  199. print(f"[{request_id}] 📊 动态超时: 300ms(较长间隔)")
  200. else:
  201. # 没有更新记录,使用保守超时
  202. timeout = 0.3
  203. print(f"[{request_id}] 📊 动态超时: 300ms(无更新记录)")
  204. # 🔥 并行执行:等待STT + 预热LLM(节省时间)
  205. final_event = self.stt_final_events.get(request_id)
  206. # 创建等待任务
  207. if final_event:
  208. stt_wait_task = asyncio.create_task(
  209. asyncio.wait_for(final_event.wait(), timeout=timeout)
  210. )
  211. else:
  212. # 降级到固定等待
  213. stt_wait_task = asyncio.create_task(asyncio.sleep(timeout))
  214. # 创建预热任务
  215. warmup_task = asyncio.create_task(warmup_llm_connection())
  216. # 并行执行(谁先完成算谁,但都不阻塞)
  217. got_final_event = False
  218. try:
  219. # 等待STT完成,同时预热在后台进行
  220. await stt_wait_task
  221. print(f"[{request_id}] ⚡ 收到STT最终结果事件!")
  222. got_final_event = True
  223. except asyncio.TimeoutError:
  224. print(f"[{request_id}] ⏰ 等待超时({int(timeout*1000)}ms),再等待50ms给最终结果一个机会...")
  225. # 🔧 关键修复:超时后再等待一小段时间,给STT最终结果到达的机会
  226. # 因为STT的最终结果可能在事件触发失败的情况下延迟到达
  227. try:
  228. if final_event:
  229. await asyncio.wait_for(final_event.wait(), timeout=0.05) # 额外等待50ms
  230. print(f"[{request_id}] ✅ 额外等待期间收到最终结果!")
  231. got_final_event = True
  232. except asyncio.TimeoutError:
  233. print(f"[{request_id}] ⏰ 额外等待也超时,使用当前结果")
  234. # 确保预热任务也完成(或超时)
  235. try:
  236. await asyncio.wait_for(warmup_task, timeout=0.1)
  237. except:
  238. pass # 预热失败不影响
  239. wait_actual = (time.time() - wait_start) * 1000
  240. print(f"[{request_id}] ✓ STT等待完成 (实际: {wait_actual:.1f}ms)")
  241. # 🔍 再次检查识别结果更新时间
  242. last_update_after = self.last_recognition_update_time.get(request_id, 0)
  243. if last_update_after and last_update:
  244. if last_update_after > last_update:
  245. new_update_delay = (last_update_after - wait_start) * 1000
  246. print(f"[{request_id}] 🔍 ✅ 等待期间有新的识别结果更新!延迟: {new_update_delay:.1f}ms")
  247. else:
  248. print(f"[{request_id}] 🔍 ⚠️ 等待期间没有新的识别结果更新")
  249. # 🔧 关键修复:如果没有收到最终结果事件,再主动等待让 STT 最终结果到达
  250. if not got_final_event:
  251. print(f"[{request_id}] 🔄 未收到最终事件,再等待500ms让识别结果更新...")
  252. # 🎯 增加等待时间:从 100ms → 500ms
  253. # 原因:commit() 后的最终结果需要更长时间(特别是在长时间没有中间结果时)
  254. for i in range(5): # 分5次检查,每次100ms
  255. await asyncio.sleep(0.1)
  256. # 检查是否有更新
  257. last_update_final = self.last_recognition_update_time.get(request_id, 0)
  258. if last_update_final > last_update_after:
  259. wait_time = (i + 1) * 100
  260. print(f"[{request_id}] ✅ 最终等待成功!识别结果已更新 (耗时: {wait_time}ms)")
  261. got_final_event = True
  262. break
  263. if not got_final_event:
  264. print(f"[{request_id}] ⚠️ 最终等待500ms后仍无更新,使用现有结果")
  265. # 获取STT识别结果
  266. text = self.recognition_results.get(request_id, "")
  267. text_length = len(text) if text else 0
  268. if not text:
  269. print(f"[{request_id}] ⚠️ STT未识别到文本,使用默认")
  270. text = "Hello, how can I help you?"
  271. else:
  272. print(f"[{request_id}] ✓ STT识别完成: \"{text}\" (长度: {text_length}字)")
  273. # 发送最终识别结果
  274. await self.send_text_result(request_id, text, True)
  275. stt_end = time.time()
  276. stt_duration = stt_end - self.request_start_times.get(request_id, stt_start)
  277. print(f"[{request_id}] ⏱️ STT总耗时: {stt_duration:.2f}秒")
  278. if request_id in self.cancelled_requests:
  279. return
  280. # 2+3. LLM流式 + TTS并行(优化:边生成边播放)
  281. print(f"[{request_id}] 步骤 2+3: LLM流式 + TTS并行")
  282. llm_start = time.time()
  283. chunk_index = 0
  284. first_audio_time = None
  285. final_emotion = "neutral"
  286. sentence_count = 0
  287. llm_first_sentence_time = None
  288. emotion_sent = False # 标记是否已发送情感
  289. # 🔥 流式处理:LLM生成一句 → 立即TTS
  290. async for sentence, emotion in ai_service.generate_response_stream(text):
  291. if request_id in self.cancelled_requests:
  292. return
  293. # 🎯 优先处理情感:第一时间发送给前端
  294. if not emotion_sent and emotion:
  295. await self.send_emotion(request_id, emotion)
  296. print(f"[{request_id}] 🎭 立即发送情感给前端: {emotion}")
  297. emotion_sent = True
  298. final_emotion = emotion
  299. # ⚡ 如果是空句子(仅传递情感),跳过 TTS 处理
  300. if not sentence:
  301. print(f"[{request_id}] ⚡ 收到仅情感标记(无句子内容),继续等待实际文本")
  302. continue
  303. sentence_count += 1
  304. # 🧹 清理文本:移除情感标记和特殊字符(保留文字内容)
  305. clean_sentence = EMOTION_TAG_PATTERN.sub('', sentence)
  306. # 移除括号符号本身,但保留括号内的文字(避免TTS发音异常)
  307. # 使用 str.translate() 比多次 replace() 更快
  308. clean_sentence = clean_sentence.translate(str.maketrans('', '', '()[]')).strip()
  309. # 如果清理后为空,跳过
  310. if not clean_sentence:
  311. print(f"[{request_id}] ⚠️ 跳过空句子(仅含情感标记)")
  312. continue
  313. # 收集AI响应(用于日志记录)
  314. self.ai_responses[request_id].append(clean_sentence)
  315. # 记录第一个句子生成时间
  316. if llm_first_sentence_time is None:
  317. llm_first_sentence_time = time.time()
  318. llm_first_sentence_duration = llm_first_sentence_time - llm_start
  319. print(f"[{request_id}] 💬 LLM第{sentence_count}句: {clean_sentence} (耗时: {llm_first_sentence_duration:.2f}秒)")
  320. else:
  321. print(f"[{request_id}] 💬 LLM第{sentence_count}句: {clean_sentence}")
  322. print(f"[{request_id}] 🎵 立即启动TTS合成第{sentence_count}句...")
  323. tts_sentence_start = time.time()
  324. # 立即TTS合成这个句子(使用清理后的文本 + 情感参数)
  325. async for audio_chunk in ai_service.text_to_speech_stream(clean_sentence, emotion):
  326. if request_id in self.cancelled_requests:
  327. return
  328. # Base64编码
  329. audio_base64 = base64.b64encode(audio_chunk).decode()
  330. # 发送音频数据
  331. await self.send_audio_response(
  332. request_id,
  333. audio_base64,
  334. chunk_index,
  335. False,
  336. )
  337. # 记录第一个音频块发送时间
  338. if chunk_index == 0:
  339. first_audio_time = time.time()
  340. time_to_first_audio = first_audio_time - audio_end_time
  341. print(f"[{request_id}] 🎵 第一个音频块已发送 [耗时: {time_to_first_audio:.2f}秒 = 录音结束→第一个音频块] ⚡")
  342. chunk_index += 1
  343. tts_sentence_duration = time.time() - tts_sentence_start
  344. print(f"[{request_id}] ✓ 第{sentence_count}句TTS完成 (耗时: {tts_sentence_duration:.2f}秒)")
  345. # 更新当前情感(但不再发送,因为已在开头发送)
  346. if emotion:
  347. final_emotion = emotion
  348. # 发送TTS结束标记
  349. await self.send_audio_response(request_id, "", chunk_index, True)
  350. # 记录最后一个音频块发送时间
  351. last_audio_time = time.time()
  352. time_to_last_audio = last_audio_time - audio_end_time
  353. print(f"[{request_id}] 🎵 最后一个音频块已发送 [耗时: {time_to_last_audio:.2f}秒 = 录音结束→最后一个音频块]")
  354. # 计算总耗时
  355. total_end = time.time()
  356. total_duration = total_end - self.request_start_times.get(request_id, stt_start)
  357. llm_tts_total = total_end - llm_start
  358. # 性能汇总日志
  359. print(f"\n{'='*60}")
  360. print(f"[{request_id}] 📊 性能汇总报告(流式优化)")
  361. print(f"{'='*60}")
  362. print(f" STT 耗时: {stt_duration:.2f}秒")
  363. print(f" LLM+TTS 流式总耗时: {llm_tts_total:.2f}秒")
  364. print(f" ├─ LLM第1句: {llm_first_sentence_duration:.2f}秒" if llm_first_sentence_time else "")
  365. print(f" └─ 共{sentence_count}个句子")
  366. if first_audio_time:
  367. print(f" 🎯 录音结束→首个音频: {time_to_first_audio:.2f}秒 ⭐ [流式优化]")
  368. print(f" 🎯 录音结束→全部音频: {time_to_last_audio:.2f}秒")
  369. print(f" 总耗时: {total_duration:.2f}秒")
  370. print(f"{'='*60}")
  371. print(f" 📝 优化验证指标:")
  372. print(f" - STT等待实际耗时: {wait_actual:.0f}ms (期望: ~300ms, 前端100ms发送间隔)")
  373. print(f" - STT stop阻塞时间: {stop_blocking:.0f}ms (期望: <10ms)")
  374. print(f" - 识别文本长度: {text_length}字 (检查完整性)")
  375. if first_audio_time:
  376. baseline_time = 1.57 # 串行基准:LLM(0.77s) + TTS(0.8s)
  377. improvement = baseline_time - time_to_first_audio
  378. improvement_pct = (improvement / baseline_time) * 100
  379. print(f" - 流式优化效果: {improvement:+.2f}秒 ({improvement_pct:+.1f}%) [基准: {baseline_time}s]")
  380. print(f" - 用户感知延迟: {time_to_first_audio:.2f}秒 [目标: <0.5s]")
  381. print(f"{'='*60}")
  382. print(f"[{request_id}] ✅ 请求处理完成\n")
  383. # 📝 记录对话日志到文件
  384. try:
  385. user_text_log = text
  386. ai_response_log = ''.join(self.ai_responses.get(request_id, []))
  387. first_audio_latency_ms = (first_audio_time - audio_end_time) * 1000 if first_audio_time else None
  388. request_logger.log_request(
  389. user_text=user_text_log,
  390. ai_response=ai_response_log,
  391. first_audio_latency=first_audio_latency_ms,
  392. request_id=request_id
  393. )
  394. except Exception as log_error:
  395. print(f"[{request_id}] ⚠️ 记录日志失败: {log_error}")
  396. except Exception as e:
  397. print(f"[{request_id}] 处理错误: {e}")
  398. import traceback
  399. traceback.print_exc()
  400. # 记录错误日志
  401. try:
  402. user_text_log = self.recognition_results.get(request_id, "")
  403. request_logger.log_error(
  404. user_text=user_text_log,
  405. error_message=str(e),
  406. request_id=request_id
  407. )
  408. except:
  409. pass
  410. await self.send_error(request_id, "NETWORK_ERROR", str(e))
  411. finally:
  412. # 清理资源
  413. await self.cleanup_request(request_id)
  414. async def start_streaming_stt(self, request_id: str) -> bool:
  415. """启动流式STT识别(使用OmniRealtimeConversation API),返回是否成功"""
  416. try:
  417. stt_init_start = time.time()
  418. print(f"[{request_id}] 🔊 启动流式STT识别(OmniRealtimeConversation API)...")
  419. # 发送"正在听..."的中间状态
  420. await self.send_text_result(request_id, "Listening...", False)
  421. from dashscope.audio.qwen_omni import OmniRealtimeConversation, OmniRealtimeCallback
  422. from dashscope.audio.qwen_omni.omni_realtime import TranscriptionParams, MultiModality
  423. import dashscope
  424. # 设置API Key(根据国际版/国内版自动选择)
  425. dashscope.api_key = settings.active_dashscope_api_key
  426. # 定义回调处理类
  427. handler = self
  428. req_id = request_id
  429. class StreamCallback(OmniRealtimeCallback):
  430. def __init__(self, conversation):
  431. self.conversation = conversation
  432. self.handlers = {
  433. 'session.created': self._handle_session_created,
  434. 'conversation.item.input_audio_transcription.completed': self._handle_final_text,
  435. 'conversation.item.input_audio_transcription.text': self._handle_stash_text,
  436. 'input_audio_buffer.speech_started': lambda r: print(f'[{req_id}] ======Speech Start======'),
  437. 'input_audio_buffer.speech_stopped': lambda r: print(f'[{req_id}] ======Speech Stop======'),
  438. 'error': self._handle_error
  439. }
  440. def on_open(self):
  441. print(f"[{req_id}] 🔗 STT连接已打开")
  442. def on_close(self, code, msg):
  443. print(f"[{req_id}] 🔌 STT连接已关闭 (code: {code}, msg: {msg})")
  444. # 标记为失败,停止后续音频帧发送
  445. if code != 1000: # 1000 是正常关闭
  446. handler.recognition_failed.add(req_id)
  447. def on_event(self, response):
  448. try:
  449. event_type = response.get('type')
  450. event_handler = self.handlers.get(event_type)
  451. # 🔍 记录所有事件(用于诊断)
  452. if event_type not in ['input_audio_buffer.speech_started', 'input_audio_buffer.speech_stopped']:
  453. print(f"[{req_id}] 📡 STT事件: {event_type}")
  454. if event_handler:
  455. event_handler(response)
  456. else:
  457. # 🔍 记录未处理的事件
  458. if event_type not in ['session.created', 'session.updated', 'input_audio_buffer.committed']:
  459. print(f"[{req_id}] ⚠️ 未处理的STT事件: {event_type}")
  460. except Exception as e:
  461. print(f"[{req_id}] ❌ 事件处理错误: {e}")
  462. def _handle_session_created(self, response):
  463. print(f"[{req_id}] ✅ 会话创建: {response['session']['id']}")
  464. def _handle_final_text(self, response):
  465. text = response.get('transcript', '')
  466. # 🔍 增强日志:显示文本长度和完整响应
  467. print(f"[{req_id}] 💬 STT识别(✅最终): {text} (长度: {len(text)}字)")
  468. print(f"[{req_id}] 🔍 最终结果完整响应: {response}")
  469. # 更新识别结果
  470. handler.recognition_results[req_id] = text
  471. handler.last_recognition_update_time[req_id] = time.time()
  472. # 触发最终结果事件
  473. if req_id in handler.stt_final_events:
  474. try:
  475. handler.event_loop.call_soon_threadsafe(handler.stt_final_events[req_id].set)
  476. print(f"[{req_id}] ⚡ 触发STT最终结果事件")
  477. except Exception as e:
  478. print(f"[{req_id}] ❌ 触发事件失败: {e}")
  479. # 发送最终结果到前端
  480. if handler.event_loop:
  481. try:
  482. asyncio.run_coroutine_threadsafe(
  483. handler.send_text_result(req_id, text, False),
  484. handler.event_loop
  485. )
  486. except Exception as e:
  487. print(f"[{req_id}] ❌ 发送最终结果失败: {e}")
  488. def _handle_stash_text(self, response):
  489. # ✅ 修复:合并 text(已确认部分)和 stash(正在识别部分)
  490. confirmed_text = response.get('text', '') # 已确认的文本
  491. stash_text = response.get('stash', '') # 正在识别的文本
  492. # 合并完整文本
  493. full_text = confirmed_text + stash_text
  494. # 🔍 增强日志:显示完整文本
  495. print(f"[{req_id}] 💬 STT识别(⏳中间): {full_text} (长度: {len(full_text)}字)")
  496. print(f"[{req_id}] ├─ 已确认: {confirmed_text} ({len(confirmed_text)}字)")
  497. print(f"[{req_id}] └─ 缓冲区: {stash_text} ({len(stash_text)}字)")
  498. # 🔍 诊断:打印原始响应(只在文本变化时)
  499. if full_text and full_text != handler.recognition_results.get(req_id, ""):
  500. print(f"[{req_id}] 🔍 中间结果完整响应: {response}")
  501. # 更新中间识别结果
  502. if full_text:
  503. handler.recognition_results[req_id] = full_text
  504. handler.last_recognition_update_time[req_id] = time.time()
  505. # 发送中间结果到前端
  506. if handler.event_loop:
  507. try:
  508. asyncio.run_coroutine_threadsafe(
  509. handler.send_text_result(req_id, full_text, False),
  510. handler.event_loop
  511. )
  512. except Exception as e:
  513. print(f"[{req_id}] ❌ 发送中间结果失败: {e}")
  514. def _handle_error(self, response):
  515. error_msg = response.get('error', {}).get('message', 'Unknown error')
  516. print(f"[{req_id}] ❌ STT错误: {error_msg}")
  517. handler.recognition_failed.add(req_id)
  518. # 创建会话对象
  519. def create_conversation():
  520. ws_url = get_dashscope_ws_url()
  521. conversation = OmniRealtimeConversation(
  522. model='qwen3-asr-flash-realtime',
  523. url=ws_url,
  524. callback=StreamCallback(conversation=None) # 先传None
  525. )
  526. # 注入自身引用
  527. conversation.callback.conversation = conversation
  528. return conversation
  529. conversation = await asyncio.to_thread(create_conversation)
  530. # 连接
  531. await asyncio.to_thread(conversation.connect)
  532. # 配置转录参数
  533. transcription_params = TranscriptionParams(
  534. language='zh', # 中文识别
  535. sample_rate=16000,
  536. input_audio_format="pcm"
  537. )
  538. # 更新会话配置
  539. await asyncio.to_thread(
  540. conversation.update_session,
  541. output_modalities=[MultiModality.TEXT],
  542. enable_input_audio_transcription=True,
  543. transcription_params=transcription_params,
  544. enable_turn_detection=False # 禁用VAD,手动提交
  545. )
  546. # 保存会话对象
  547. self.recognition_sessions[request_id] = conversation
  548. # 创建最终结果事件
  549. self.stt_final_events[request_id] = asyncio.Event()
  550. stt_init_duration = (time.time() - stt_init_start) * 1000
  551. print(f"[{request_id}] ✅ STT识别会话已启动 (耗时: {stt_init_duration:.1f}ms)")
  552. # 检查是否已经失败
  553. if request_id in self.recognition_failed:
  554. print(f"[{request_id}] ❌ STT连接启动后立即失败")
  555. return False
  556. return True
  557. except Exception as e:
  558. print(f"[{request_id}] 启动STT识别失败: {e}")
  559. import traceback
  560. traceback.print_exc()
  561. return False
  562. async def stop_streaming_stt(self, request_id: str):
  563. """停止流式STT识别(异步非阻塞)- OmniRealtimeConversation版本"""
  564. try:
  565. conversation = self.recognition_sessions.get(request_id)
  566. if conversation:
  567. print(f"[{request_id}] 🛑 提交音频并触发STT异步停止...")
  568. # 🔥 优化:异步提交并关闭,不等待完成
  569. async def _async_stop():
  570. try:
  571. stop_start = time.time()
  572. # 先提交音频(触发最终识别)
  573. await asyncio.to_thread(conversation.commit)
  574. print(f"[{request_id}] ✅ 音频已提交")
  575. # 🎯 增加等待时间:从 50ms → 100ms
  576. # 让 commit() 后的最终结果有更多时间到达
  577. await asyncio.sleep(0.1)
  578. # 关闭连接
  579. await asyncio.to_thread(conversation.close)
  580. stop_duration = (time.time() - stop_start) * 1000
  581. print(f"[{request_id}] ✅ STT连接已关闭 (耗时: {stop_duration:.1f}ms)")
  582. except Exception as e:
  583. print(f"[{request_id}] ⚠️ STT停止异常: {e}")
  584. # 启动异步任务但不等待
  585. asyncio.create_task(_async_stop())
  586. print(f"[{request_id}] ⚡ STT异步停止已触发,继续处理...")
  587. except Exception as e:
  588. print(f"[{request_id}] ❌ 停止STT识别错误: {e}")
  589. async def cleanup_request(self, request_id: str):
  590. """清理请求相关资源"""
  591. # 清理audio buffer
  592. if request_id in self.audio_buffers:
  593. del self.audio_buffers[request_id]
  594. # 清理识别会话
  595. if request_id in self.recognition_sessions:
  596. del self.recognition_sessions[request_id]
  597. # 清理识别结果
  598. if request_id in self.recognition_results:
  599. del self.recognition_results[request_id]
  600. # 清理取消标记
  601. if request_id in self.cancelled_requests:
  602. self.cancelled_requests.remove(request_id)
  603. # 清理失败标记
  604. if request_id in self.recognition_failed:
  605. self.recognition_failed.remove(request_id)
  606. # 清理时间记录
  607. if request_id in self.request_start_times:
  608. del self.request_start_times[request_id]
  609. if request_id in self.last_audio_frame_time:
  610. del self.last_audio_frame_time[request_id]
  611. if request_id in self.last_recognition_update_time:
  612. del self.last_recognition_update_time[request_id]
  613. if request_id in self.audio_end_times:
  614. del self.audio_end_times[request_id]
  615. # 清理AI响应收集器
  616. if request_id in self.ai_responses:
  617. del self.ai_responses[request_id]
  618. # 清理事件
  619. if request_id in self.stt_final_events:
  620. del self.stt_final_events[request_id]
  621. print(f"[{request_id}] 资源已清理")
  622. async def handle_cancel(self, message: CancelMessage):
  623. """处理取消请求"""
  624. request_id = message.requestId
  625. print(f"[{request_id}] 收到取消请求")
  626. self.cancelled_requests.add(request_id)
  627. # 停止STT识别
  628. await self.stop_streaming_stt(request_id)
  629. # 清理资源
  630. await self.cleanup_request(request_id)
  631. async def handle_ping(self, message: PingMessage):
  632. """处理Ping"""
  633. await self.send_pong(message.requestId)
  634. async def send_text_result(self, request_id: str, text: str, is_final: bool):
  635. """发送文本结果"""
  636. message = TextResultMessage(
  637. type="text_result",
  638. requestId=request_id,
  639. timestamp=self._get_timestamp(),
  640. text=text,
  641. isFinal=is_final,
  642. )
  643. await self.websocket.send_text(message.model_dump_json())
  644. async def send_emotion(self, request_id: str, emotion: str):
  645. """发送情感标签"""
  646. message = EmotionMessage(
  647. type="emotion",
  648. requestId=request_id,
  649. timestamp=self._get_timestamp(),
  650. emotion=emotion,
  651. )
  652. await self.websocket.send_text(message.model_dump_json())
  653. async def send_audio_response(
  654. self, request_id: str, data: str, chunk_index: int, is_last: bool,
  655. format: str = "pcm", sample_rate: int = 24000
  656. ):
  657. """发送音频响应"""
  658. message = AudioResponseMessage(
  659. type="audio_response",
  660. requestId=request_id,
  661. timestamp=self._get_timestamp(),
  662. data=data,
  663. sampleRate=sample_rate,
  664. format=format,
  665. chunkIndex=chunk_index,
  666. isLast=is_last,
  667. )
  668. await self.websocket.send_text(message.model_dump_json())
  669. async def send_error(self, request_id: str, code: str, error_message: str):
  670. """发送错误消息"""
  671. message = ErrorMessage(
  672. type="error",
  673. requestId=request_id,
  674. timestamp=self._get_timestamp(),
  675. code=code,
  676. message=error_message,
  677. )
  678. await self.websocket.send_text(message.model_dump_json())
  679. async def send_pong(self, request_id: str):
  680. """发送Pong"""
  681. message = PongMessage(
  682. type="pong",
  683. requestId=request_id,
  684. timestamp=self._get_timestamp(),
  685. )
  686. await self.websocket.send_text(message.model_dump_json())
  687. def _get_timestamp(self) -> int:
  688. """获取当前时间戳(毫秒)"""
  689. return int(time.time() * 1000)