heartbeat_monitor.py 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191
  1. """
  2. 心跳监控服务
  3. """
  4. import time
  5. from typing import Dict, Optional
  6. from datetime import datetime, timedelta
  7. from sqlalchemy.orm import Session
  8. from app.database import get_db
  9. from app.models.user import User
  10. from app.models.session import Session
  11. from app.models.system_config import SystemConfig
  12. from app.services.session_service import SessionService
  13. from app.core.logger import logger
  14. from app.utils.timezone import beijing_now
  15. class HeartbeatMonitor:
  16. """心跳监控服务"""
  17. # 内存缓存心跳状态,避免频繁数据库查询
  18. _heartbeat_cache: Dict[int, Dict] = {}
  19. _last_cleanup: float = 0
  20. @staticmethod
  21. def record_heartbeat_success(db: Session, user_id: int, session_id: str) -> None:
  22. """记录心跳成功"""
  23. try:
  24. cache_key = f"{user_id}:{session_id}"
  25. # 更新内存缓存
  26. if cache_key not in HeartbeatMonitor._heartbeat_cache:
  27. HeartbeatMonitor._heartbeat_cache[cache_key] = {
  28. "user_id": user_id,
  29. "session_id": session_id,
  30. "failure_count": 0,
  31. "last_success": time.time(),
  32. "last_failure": 0,
  33. "is_protected": False
  34. }
  35. else:
  36. cache_entry = HeartbeatMonitor._heartbeat_cache[cache_key]
  37. cache_entry["failure_count"] = 0 # 重置失败计数
  38. cache_entry["last_success"] = time.time()
  39. cache_entry["is_protected"] = False
  40. logger.debug(f"用户 {user_id} 心跳成功,失败计数已重置")
  41. except Exception as e:
  42. logger.exception("记录心跳成功失败")
  43. @staticmethod
  44. def record_heartbeat_failure(db: Session, user_id: int, session_id: str) -> bool:
  45. """
  46. 记录心跳失败
  47. Returns:
  48. bool: 是否触发了保护措施
  49. """
  50. try:
  51. cache_key = f"{user_id}:{session_id}"
  52. # 使用默认的最大失败次数(可以后续从配置中获取)
  53. max_failures = 3
  54. # 更新内存缓存
  55. if cache_key not in HeartbeatMonitor._heartbeat_cache:
  56. HeartbeatMonitor._heartbeat_cache[cache_key] = {
  57. "user_id": user_id,
  58. "session_id": session_id,
  59. "failure_count": 1,
  60. "last_success": 0,
  61. "last_failure": time.time(),
  62. "is_protected": False
  63. }
  64. else:
  65. cache_entry = HeartbeatMonitor._heartbeat_cache[cache_key]
  66. cache_entry["failure_count"] += 1
  67. cache_entry["last_failure"] = time.time()
  68. failure_count = HeartbeatMonitor._heartbeat_cache[cache_key]["failure_count"]
  69. logger.warning(f"用户 {user_id} 心跳失败 {failure_count}/{max_failures}")
  70. # 检查是否需要触发保护措施
  71. if failure_count >= max_failures:
  72. HeartbeatMonitor._execute_protection_measures(db, user_id, session_id)
  73. HeartbeatMonitor._heartbeat_cache[cache_key]["is_protected"] = True
  74. return True
  75. return False
  76. except Exception as e:
  77. logger.exception("记录心跳失败异常")
  78. return False
  79. @staticmethod
  80. def _execute_protection_measures(db: Session, user_id: int, session_id: str) -> None:
  81. """执行保护措施"""
  82. try:
  83. logger.warning(f"执行用户 {user_id} 的保护措施")
  84. # 1. 停用所有相关会话
  85. deactivated_count = SessionService.deactivate_user_sessions(db, user_id)
  86. if deactivated_count > 0:
  87. logger.info(f"已停用用户 {user_id} 的 {deactivated_count} 个会话")
  88. # 2. 可以在这里添加其他保护措施
  89. # 例如:通知其他相关服务、记录安全事件等
  90. # 3. 记录保护措施执行
  91. logger.warning(f"用户 {user_id} 因心跳失败过多,已执行保护措施")
  92. except Exception as e:
  93. logger.exception("执行保护措施失败")
  94. @staticmethod
  95. def get_heartbeat_status(user_id: int, session_id: str) -> Dict:
  96. """获取心跳状态"""
  97. cache_key = f"{user_id}:{session_id}"
  98. return HeartbeatMonitor._heartbeat_cache.get(cache_key, {
  99. "user_id": user_id,
  100. "session_id": session_id,
  101. "failure_count": 0,
  102. "last_success": 0,
  103. "last_failure": 0,
  104. "is_protected": False
  105. })
  106. @staticmethod
  107. def cleanup_cache() -> int:
  108. """
  109. 清理过期的缓存数据
  110. Returns:
  111. int: 清理的条目数量
  112. """
  113. try:
  114. current_time = time.time()
  115. expired_keys = []
  116. # 查找过期条目(超过1小时没有活动的)
  117. for cache_key, cache_entry in HeartbeatMonitor._heartbeat_cache.items():
  118. last_activity = max(cache_entry["last_success"], cache_entry["last_failure"])
  119. if current_time - last_activity > 3600: # 1小时
  120. expired_keys.append(cache_key)
  121. # 删除过期条目
  122. for key in expired_keys:
  123. del HeartbeatMonitor._heartbeat_cache[key]
  124. if expired_keys:
  125. logger.info(f"清理了 {len(expired_keys)} 个过期的缓存条目")
  126. return len(expired_keys)
  127. except Exception as e:
  128. logger.exception("清理缓存失败")
  129. return 0
  130. @staticmethod
  131. def get_monitoring_stats() -> Dict:
  132. """获取监控统计信息"""
  133. try:
  134. total_users = len(HeartbeatMonitor._heartbeat_cache)
  135. protected_users = sum(1 for entry in HeartbeatMonitor._heartbeat_cache.values()
  136. if entry["is_protected"])
  137. failing_users = sum(1 for entry in HeartbeatMonitor._heartbeat_cache.values()
  138. if entry["failure_count"] > 0)
  139. return {
  140. "total_monitored_users": total_users,
  141. "protected_users": protected_users,
  142. "failing_users": failing_users,
  143. "healthy_users": total_users - failing_users,
  144. "cache_size": len(HeartbeatMonitor._heartbeat_cache)
  145. }
  146. except Exception as e:
  147. logger.exception("获取监控统计失败")
  148. return {}
  149. @staticmethod
  150. def reset_user_heartbeat(user_id: int, session_id: str) -> None:
  151. """重置用户的缓存状态(用于重新连接时)"""
  152. try:
  153. cache_key = f"{user_id}:{session_id}"
  154. if cache_key in HeartbeatMonitor._heartbeat_cache:
  155. del HeartbeatMonitor._heartbeat_cache[cache_key]
  156. logger.info(f"已重置用户 {user_id} 的心跳缓存")
  157. except Exception as e:
  158. logger.exception("重置用户心跳缓存失败")