sync_float_price.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713
  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. BUFF饰品磨损区间价格数据同步脚本(修正数据逻辑)
  5. 功能:
  6. 1. 将有tag_id、tag_name、min_float、max_float的详细价格数据存储到float_price表
  7. 2. 将没有这些属性的基本价格数据存储到float_price_attr表
  8. 3. 周期调度由 run_buff_sync_loop.sh 统一控制
  9. 使用方法:
  10. python sync_float_price.py # 执行一次同步
  11. 作者: 游铛铛技术数字员工
  12. 创建时间: 2026-01-28
  13. """
  14. import os
  15. import sys
  16. import time
  17. import logging
  18. import json
  19. import requests
  20. import pymysql
  21. from datetime import datetime, timedelta
  22. from sync_db_errors import is_database_connection_error
  23. from typing import List, Dict, Any, Optional, Tuple, Callable
  24. import urllib.parse
  25. # ============================================================
  26. # 配置信息
  27. # ============================================================
  28. # BUFF API配置
  29. BUFF_FLOAT_API_URL = "https://buff.163.com/api/market/enterprise/float/items"
  30. BUFF_API_TOKEN = "c4a1ebaf64d64d1a9a320bc2fcfa1f7e"
  31. # 数据库配置(从环境变量读取)
  32. DB_HOST = os.getenv("DB_HOST", "47.99.164.66")
  33. DB_PORT = int(os.getenv("DB_PORT", "3306"))
  34. DB_USER = os.getenv("DB_USER", "root")
  35. DB_PASSWORD = os.getenv("DB_PASSWORD", "hn02le.34lkdLKD")
  36. DB_NAME = os.getenv("DB_NAME", "buff")
  37. # 请求配置
  38. REQUEST_TIMEOUT = 60
  39. # 批量插入:单批过大或网络慢时易触发超时;可通过环境变量调小单批或调大超时
  40. BATCH_SIZE = int(os.getenv("BULK_INSERT_BATCH_SIZE", "5000"))
  41. MAX_RETRIES = 3
  42. RETRY_DELAY = 5
  43. # 大批量写入建议 ≥300s,避免 PyMySQL 在长时间读写时触发 read/write 超时(默认 30 过短)
  44. DB_READ_TIMEOUT = int(os.getenv("DB_READ_TIMEOUT", "600"))
  45. DB_WRITE_TIMEOUT = int(os.getenv("DB_WRITE_TIMEOUT", "600"))
  46. DB_BULK_INSERT_RETRIES = int(os.getenv("DB_BULK_INSERT_RETRIES", "3"))
  47. # ============================================================
  48. # 配置日志
  49. # ============================================================
  50. def setup_logging():
  51. log_dir = os.path.dirname(os.path.abspath(__file__))
  52. log_file = os.path.join(log_dir, "logs", f"sync_float_price_{datetime.now().strftime('%Y%m%d')}.log")
  53. os.makedirs(os.path.dirname(log_file), exist_ok=True)
  54. logging.basicConfig(
  55. level=logging.INFO,
  56. format='%(asctime)s - %(levelname)s - %(message)s',
  57. handlers=[
  58. logging.StreamHandler(sys.stdout),
  59. logging.FileHandler(log_file, encoding='utf-8')
  60. ]
  61. )
  62. return logging.getLogger(__name__)
  63. logger = setup_logging()
  64. # ============================================================
  65. # 数据库操作类
  66. # ============================================================
  67. class FloatPriceDatabase:
  68. """磨损区间价格数据库操作类"""
  69. def __init__(self):
  70. self.connection = None
  71. self._connect()
  72. def _connect(self):
  73. """建立数据库连接"""
  74. try:
  75. self.connection = pymysql.connect(
  76. host=DB_HOST,
  77. port=DB_PORT,
  78. user=DB_USER,
  79. password=DB_PASSWORD,
  80. database=DB_NAME,
  81. charset='utf8mb4',
  82. cursorclass=pymysql.cursors.DictCursor,
  83. autocommit=False,
  84. connect_timeout=10,
  85. read_timeout=DB_READ_TIMEOUT,
  86. write_timeout=DB_WRITE_TIMEOUT,
  87. )
  88. logger.info(f"数据库连接成功: {DB_HOST}:{DB_PORT}/{DB_NAME}")
  89. except Exception as e:
  90. logger.error(f"数据库连接失败: {e}")
  91. raise
  92. def _reconnect(self):
  93. """连接丢失或失效时重建连接(供批量写入重试用)。"""
  94. try:
  95. if self.connection:
  96. self.connection.close()
  97. except Exception:
  98. pass
  99. self.connection = None
  100. self._connect()
  101. @staticmethod
  102. def _is_transient_db_error(exc: BaseException) -> bool:
  103. """可重试的数据库错误:断连、服务端重启中等。"""
  104. if not isinstance(exc, pymysql.err.OperationalError):
  105. return False
  106. code = exc.args[0] if exc.args else 0
  107. # 2006 lost connection, 2013 lost during query, 2014 commands out of sync,
  108. # 1053 server shutdown, 2003 can't connect
  109. return code in (2006, 2013, 2014, 1053, 2003)
  110. def _safe_rollback(self) -> None:
  111. """连接已断开时避免 rollback 再抛错掩盖原始异常。"""
  112. if not self.connection:
  113. return
  114. try:
  115. if getattr(self.connection, "open", False):
  116. self.connection.rollback()
  117. except Exception as e:
  118. logger.warning(f"rollback 失败(连接可能已断开): {e}")
  119. def execute_with_transient_retry(self, label: str, fn: Callable[[], Any]) -> Any:
  120. """对断连、服务端关闭等可重试错误:安全回滚、重连后重新执行整段逻辑(如先 TRUNCATE 再 INSERT)。"""
  121. attempt = 0
  122. while True:
  123. try:
  124. return fn()
  125. except pymysql.err.OperationalError as e:
  126. attempt += 1
  127. self._safe_rollback()
  128. if not self._is_transient_db_error(e) or attempt > DB_BULK_INSERT_RETRIES:
  129. raise
  130. logger.warning(
  131. "%s 失败 (%s/%s),将重连后重试: %s",
  132. label,
  133. attempt,
  134. DB_BULK_INSERT_RETRIES,
  135. e,
  136. )
  137. time.sleep(RETRY_DELAY * attempt)
  138. self._reconnect()
  139. except Exception:
  140. self._safe_rollback()
  141. raise
  142. def close(self):
  143. if self.connection and self.connection.open:
  144. self.connection.close()
  145. logger.info("数据库连接已关闭")
  146. def clear_table(self, table_name: str):
  147. """清空表"""
  148. try:
  149. with self.connection.cursor() as cursor:
  150. cursor.execute(f"TRUNCATE TABLE {table_name}")
  151. self.connection.commit()
  152. logger.info(f"已清空 {table_name} 表")
  153. except Exception as e:
  154. logger.warning(f"TRUNCATE失败,使用DELETE: {e}")
  155. try:
  156. with self.connection.cursor() as cursor:
  157. cursor.execute(f"DELETE FROM {table_name}")
  158. self.connection.commit()
  159. logger.info(f"已清空 {table_name} 表")
  160. except Exception as e2:
  161. logger.error(f"清空表失败: {e2}")
  162. raise
  163. def _bulk_insert_one_batch(
  164. self,
  165. cursor,
  166. sql: str,
  167. batch: List[Dict[str, Any]],
  168. columns: List[str],
  169. ) -> None:
  170. values = []
  171. for item in batch:
  172. row_values = []
  173. for col in columns:
  174. value = item.get(col)
  175. if value is None:
  176. row_values.append(None)
  177. elif isinstance(value, datetime):
  178. row_values.append(value.strftime('%Y-%m-%d %H:%M:%S'))
  179. elif isinstance(value, (int, float)):
  180. row_values.append(value)
  181. else:
  182. row_values.append(str(value))
  183. values.append(tuple(row_values))
  184. cursor.executemany(sql, values)
  185. def _safe_restore_session_defaults(self, cursor) -> None:
  186. """连接已断开时避免在 finally 里再次抛错掩盖原始异常。"""
  187. if not self.connection:
  188. return
  189. try:
  190. if getattr(self.connection, "open", False):
  191. cursor.execute("SET SESSION autocommit = 1")
  192. cursor.execute("SET SESSION unique_checks = 1")
  193. cursor.execute("SET SESSION foreign_key_checks = 1")
  194. except Exception as e:
  195. logger.warning(f"恢复会话设置失败(连接可能已断开),可忽略: {e}")
  196. def bulk_insert(self, table_name: str, data_list: List[Dict[str, Any]],
  197. columns: List[str]) -> int:
  198. """高性能批量插入;单批失败时对断连/服务端关闭类错误自动重连并重试当前批。"""
  199. if not data_list:
  200. return 0
  201. column_list = ', '.join([f'`{col}`' for col in columns])
  202. placeholders = ', '.join(['%s'] * len(columns))
  203. sql = f"INSERT INTO `{table_name}` ({column_list}) VALUES ({placeholders})"
  204. total_inserted = 0
  205. cursor = self.connection.cursor()
  206. cursor.execute("SET SESSION autocommit = 0")
  207. cursor.execute("SET SESSION unique_checks = 0")
  208. cursor.execute("SET SESSION foreign_key_checks = 0")
  209. try:
  210. for i in range(0, len(data_list), BATCH_SIZE):
  211. batch = data_list[i:i + BATCH_SIZE]
  212. attempt = 0
  213. while True:
  214. try:
  215. self._bulk_insert_one_batch(cursor, sql, batch, columns)
  216. break
  217. except pymysql.err.OperationalError as e:
  218. attempt += 1
  219. if not self._is_transient_db_error(e) or attempt > DB_BULK_INSERT_RETRIES:
  220. raise
  221. logger.warning(
  222. "批量插入单批失败 (%s/%s),将重连后重试当前批: %s",
  223. attempt,
  224. DB_BULK_INSERT_RETRIES,
  225. e,
  226. )
  227. time.sleep(RETRY_DELAY * attempt)
  228. self._reconnect()
  229. cursor = self.connection.cursor()
  230. cursor.execute("SET SESSION autocommit = 0")
  231. cursor.execute("SET SESSION unique_checks = 0")
  232. cursor.execute("SET SESSION foreign_key_checks = 0")
  233. total_inserted += len(batch)
  234. # 每批提交,缩短长事务、降低单次失败回滚粒度(中间表下一轮会 TRUNCATE)
  235. self.connection.commit()
  236. if i > 0 and i % (BATCH_SIZE * 3) == 0:
  237. logger.info(f"批量插入进度: {total_inserted}/{len(data_list)}")
  238. finally:
  239. self._safe_restore_session_defaults(cursor)
  240. try:
  241. cursor.close()
  242. except Exception:
  243. pass
  244. return total_inserted
  245. def insert_float_price_middle(self, data_list: List[Dict[str, Any]]) -> int:
  246. """批量插入数据到buff_goods_float_price_middle表(存储有tag和磨损区间的详细数据)"""
  247. if not data_list:
  248. return 0
  249. # 定义列和对应的数据字段
  250. columns = [
  251. 'goods_id', 'name', 'qj_min_price', 'qj_buy_max_price',
  252. 'qj_avg_min_price', 'qj_sell_num', 'qj_buy_order_num',
  253. 'min_float', 'max_float', 'tag_id', 'tag_name',
  254. 'stat_time', 'create_time'
  255. ]
  256. # 准备数据
  257. now = datetime.now()
  258. processed_data = []
  259. logger.info(f"开始准备 {len(data_list)} 条详细价格数据(有磨损区间和标签)...")
  260. start_time = time.time()
  261. for item in data_list:
  262. processed_data.append({
  263. 'goods_id': str(item.get('goods_id', '')),
  264. 'name': item.get('name', ''),
  265. 'qj_min_price': float(item.get('min_price', 0)) if item.get('min_price') else 0.00,
  266. 'qj_buy_max_price': float(item.get('buy_max_price', 0)) if item.get('buy_max_price') else 0.00,
  267. 'qj_avg_min_price': float(item.get('avg_min_price', 0)) if item.get('avg_min_price') else 0.00,
  268. 'qj_sell_num': int(item.get('sell_num', 0)) if item.get('sell_num') else 0,
  269. 'qj_buy_order_num': int(item.get('buy_order_num', 0)) if item.get('buy_order_num') else 0,
  270. 'min_float': float(item.get('min_float', 0)) if item.get('min_float') is not None else 0.000000,
  271. 'max_float': float(item.get('max_float', 0)) if item.get('max_float') is not None else 0.000000,
  272. 'tag_id': item.get('tag_id'),
  273. 'tag_name': item.get('tag_name'),
  274. 'stat_time': datetime.fromtimestamp(item.get('stat_time', 0)),
  275. 'create_time': now
  276. })
  277. prep_time = time.time() - start_time
  278. logger.info(f"数据准备完成,耗时: {prep_time:.2f}秒")
  279. logger.info(f"开始插入 {len(processed_data)} 条数据到buff_goods_float_price_middle...")
  280. insert_start = time.time()
  281. result = self.bulk_insert('buff_goods_float_price_middle', processed_data, columns)
  282. insert_time = time.time() - insert_start
  283. logger.info(f"插入完成: {result} 条,耗时: {insert_time:.2f}秒")
  284. return result
  285. def copy_from_middle_to_float_price(self) -> int:
  286. """将中间表数据插入到buff_goods_float_price表(先清空再插入)"""
  287. def _do_copy() -> int:
  288. self.clear_table("buff_goods_float_price")
  289. with self.connection.cursor() as cursor:
  290. sql = """
  291. INSERT INTO buff_goods_float_price
  292. (goods_id, name, qj_min_price, qj_buy_max_price, qj_avg_min_price,
  293. qj_sell_num, qj_buy_order_num, min_float, max_float, tag_id, tag_name,
  294. stat_time, create_time, update_time)
  295. SELECT goods_id, name, qj_min_price, qj_buy_max_price, qj_avg_min_price,
  296. qj_sell_num, qj_buy_order_num, min_float, max_float, tag_id, tag_name,
  297. stat_time, NOW(), NOW()
  298. FROM buff_goods_float_price_middle
  299. """
  300. cursor.execute(sql)
  301. affected_rows = cursor.connection.affected_rows()
  302. self.connection.commit()
  303. return affected_rows
  304. try:
  305. return self.execute_with_transient_retry(
  306. "复制详细价格到 buff_goods_float_price",
  307. _do_copy,
  308. )
  309. except Exception as e:
  310. logger.error(f"复制详细价格数据失败: {e}")
  311. raise
  312. def insert_float_price_attr_middle(self, data_list: List[Dict[str, Any]]) -> int:
  313. """批量插入数据到buff_goods_float_price_attr_middle表(存储没有tag和磨损区间的基本数据)"""
  314. if not data_list:
  315. return 0
  316. # 定义列和对应的数据字段
  317. columns = [
  318. 'goods_id', 'name', 'qj_min_price', 'qj_buy_max_price',
  319. 'qj_avg_min_price', 'qj_sell_num', 'qj_buy_order_num',
  320. 'stat_time', 'create_time'
  321. ]
  322. # 准备数据
  323. now = datetime.now()
  324. processed_data = []
  325. logger.info(f"开始准备 {len(data_list)} 条基本价格数据(无磨损区间无标签)...")
  326. start_time = time.time()
  327. for item in data_list:
  328. processed_data.append({
  329. 'goods_id': str(item.get('goods_id', '')),
  330. 'name': item.get('name', ''),
  331. 'qj_min_price': float(item.get('min_price', 0)) if item.get('min_price') else 0.00,
  332. 'qj_buy_max_price': float(item.get('buy_max_price', 0)) if item.get('buy_max_price') else 0.00,
  333. 'qj_avg_min_price': float(item.get('avg_min_price', 0)) if item.get('avg_min_price') else 0.00,
  334. 'qj_sell_num': int(item.get('sell_num', 0)) if item.get('sell_num') else 0,
  335. 'qj_buy_order_num': int(item.get('buy_order_num', 0)) if item.get('buy_order_num') else 0,
  336. 'stat_time': datetime.fromtimestamp(item.get('stat_time', 0)),
  337. 'create_time': now
  338. })
  339. prep_time = time.time() - start_time
  340. logger.info(f"数据准备完成,耗时: {prep_time:.2f}秒")
  341. logger.info(f"开始插入 {len(processed_data)} 条数据到buff_goods_float_price_attr_middle...")
  342. insert_start = time.time()
  343. result = self.bulk_insert('buff_goods_float_price_attr_middle', processed_data, columns)
  344. insert_time = time.time() - insert_start
  345. logger.info(f"插入完成: {result} 条,耗时: {insert_time:.2f}秒")
  346. return result
  347. def copy_from_middle_to_float_price_attr(self) -> int:
  348. """将中间表数据插入到buff_goods_float_price_attr表(先清空再插入)"""
  349. def _do_copy() -> int:
  350. self.clear_table("buff_goods_float_price_attr")
  351. with self.connection.cursor() as cursor:
  352. sql = """
  353. INSERT INTO buff_goods_float_price_attr
  354. (goods_id, name, qj_min_price, qj_buy_max_price, qj_avg_min_price,
  355. qj_sell_num, qj_buy_order_num, stat_time, create_time, update_time)
  356. SELECT goods_id, name, qj_min_price, qj_buy_max_price, qj_avg_min_price,
  357. qj_sell_num, qj_buy_order_num, stat_time, NOW(), NOW()
  358. FROM buff_goods_float_price_attr_middle
  359. """
  360. cursor.execute(sql)
  361. affected_rows = cursor.connection.affected_rows()
  362. self.connection.commit()
  363. return affected_rows
  364. try:
  365. return self.execute_with_transient_retry(
  366. "复制基本价格到 buff_goods_float_price_attr",
  367. _do_copy,
  368. )
  369. except Exception as e:
  370. logger.error(f"复制基本价格数据失败: {e}")
  371. raise
  372. # ============================================================
  373. # BUFF API 操作类
  374. # ============================================================
  375. class BuffFloatPriceAPI:
  376. """BUFF磨损区间价格API操作类"""
  377. def __init__(self):
  378. self.session = requests.Session()
  379. self.session.headers.update({
  380. "Authorization": f"Bearer {BUFF_API_TOKEN}",
  381. "Accept": "application/json",
  382. "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
  383. })
  384. def fetch_float_price_data(self, game: str = "cs2", include_sticker: int = 1) -> Dict[str, Any]:
  385. """获取磨损区间价格数据"""
  386. params = {"game": game, "include_sticker": include_sticker}
  387. for retry in range(MAX_RETRIES):
  388. try:
  389. logger.info(f"请求BUFF磨损区间API (第{retry + 1}次尝试)...")
  390. response = self.session.get(BUFF_FLOAT_API_URL, params=params, timeout=REQUEST_TIMEOUT)
  391. response.raise_for_status()
  392. data = response.json()
  393. logger.info("=" * 60)
  394. logger.info("🔍 磨损区间API返回数据结构:")
  395. logger.info(f" 顶层keys: {list(data.keys())}")
  396. # 获取数据统计信息
  397. stat_time = data.get("stat_time", 0)
  398. remaining = data.get("req_remaining", 0)
  399. info_data = data.get("info", [])
  400. logger.info(f" 统计时间: {datetime.fromtimestamp(stat_time)}")
  401. logger.info(f" 剩余调用次数: {remaining}")
  402. logger.info(f" 数据条数: {len(info_data)}")
  403. # 检查数据格式
  404. if info_data and len(info_data) > 0:
  405. first_item = info_data[0]
  406. logger.info(f" 第一条数据检查:")
  407. logger.info(f" goods_id: {first_item.get('goods_id')}")
  408. logger.info(f" name: {first_item.get('name')}")
  409. sales = first_item.get('sales', [])
  410. logger.info(f" sales数量: {len(sales)}")
  411. if sales:
  412. for i, sale in enumerate(sales[:2]): # 显示前2个sales
  413. logger.info(f" 第{i+1}个sales:")
  414. logger.info(f" 字段: {list(sale.keys())}")
  415. logger.info(f" min_float: {sale.get('min_float')}")
  416. logger.info(f" max_float: {sale.get('max_float')}")
  417. logger.info(f" tag_id: {sale.get('tag_id')}")
  418. logger.info(f" tag_name: {sale.get('tag_name')}")
  419. logger.info("=" * 60)
  420. # 验证数据格式
  421. if "info" not in data:
  422. logger.error(f"API返回格式错误: 无info字段, 完整响应: {data}")
  423. raise Exception(f"API错误: {data.get('msg', '未知错误')}")
  424. logger.info(f"API请求成功,获取到 {len(info_data)} 条饰品记录")
  425. logger.info(f"API剩余调用次数: {remaining}")
  426. return data
  427. except requests.exceptions.RequestException as e:
  428. logger.error(f"API请求失败 (尝试 {retry + 1}/{MAX_RETRIES}): {e}")
  429. if 'response' in locals():
  430. logger.error(f"响应状态码: {response.status_code}")
  431. logger.error(f"响应内容: {response.text[:500]}")
  432. if retry < MAX_RETRIES - 1:
  433. time.sleep(RETRY_DELAY)
  434. else:
  435. raise
  436. def parse_float_price_data(self, api_response: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
  437. """解析磨损区间价格数据,返回两种类型的数据列表
  438. Returns:
  439. tuple: (detailed_price_list, basic_price_list)
  440. detailed_price_list: 包含有tag和磨损区间的详细价格数据(用于buff_goods_float_price表)
  441. basic_price_list: 不包含tag和磨损区间的基本价格数据(用于buff_goods_float_price_attr表)
  442. """
  443. logger.info("🔍 开始解析磨损区间数据...")
  444. stat_time = api_response.get("stat_time", 0)
  445. info_data = api_response.get("info", [])
  446. logger.info(f" stat_time: {datetime.fromtimestamp(stat_time)}")
  447. logger.info(f" info列表长度: {len(info_data)}")
  448. detailed_price_list = [] # 包含有tag和磨损区间的详细数据
  449. basic_price_list = [] # 不包含tag和磨损区间的基本数据
  450. start_time = time.time()
  451. total_detailed_count = 0
  452. total_basic_count = 0
  453. for item in info_data:
  454. goods_id = item.get("goods_id")
  455. name = item.get("name")
  456. sales = item.get("sales", [])
  457. # 标记是否已找到基本价格数据
  458. found_basic_price = False
  459. # 处理每个sales条目
  460. for sale in sales:
  461. # 检查是否有tag和磨损区间
  462. has_tag = sale.get("tag_id") is not None or sale.get("tag_name") is not None
  463. has_float = sale.get("min_float") is not None or sale.get("max_float") is not None
  464. # 构建销售数据
  465. sale_data = {
  466. "goods_id": goods_id,
  467. "name": name,
  468. "min_price": sale.get("min_price"),
  469. "avg_min_price": sale.get("avg_min_price"),
  470. "buy_max_price": sale.get("buy_max_price"),
  471. "sell_num": sale.get("sell_num"),
  472. "buy_order_num": sale.get("buy_order_num"),
  473. "stat_time": stat_time,
  474. "min_float": sale.get("min_float"),
  475. "max_float": sale.get("max_float"),
  476. "tag_id": sale.get("tag_id"),
  477. "tag_name": sale.get("tag_name")
  478. }
  479. # 如果有tag或磨损区间,则添加到详细价格列表
  480. if has_tag or has_float:
  481. total_detailed_count += 1
  482. detailed_price_list.append(sale_data)
  483. # 否则添加到基本价格列表(每个饰品只取第一个没有tag和磨损区间的数据)
  484. elif not found_basic_price:
  485. total_basic_count += 1
  486. found_basic_price = True
  487. basic_price_list.append(sale_data)
  488. elapsed = time.time() - start_time
  489. logger.info(f" 解析完成,耗时: {elapsed:.2f}秒")
  490. logger.info(f" 总饰品记录: {len(info_data)} 条")
  491. logger.info(f" 详细价格数据(有tag或有磨损): {total_detailed_count} 条")
  492. logger.info(f" 基本价格数据(无tag无磨损): {total_basic_count} 条")
  493. # 统计不同类型的数据
  494. if detailed_price_list:
  495. logger.info(f" 详细价格数据样本:")
  496. for i in range(min(2, len(detailed_price_list))):
  497. sample = detailed_price_list[i]
  498. logger.info(f" [{i}] goods_id: {sample.get('goods_id')}, "
  499. f"min_float: {sample.get('min_float')}, tag_id: {sample.get('tag_id')}")
  500. if basic_price_list:
  501. logger.info(f" 基本价格数据样本:")
  502. for i in range(min(2, len(basic_price_list))):
  503. sample = basic_price_list[i]
  504. logger.info(f" [{i}] goods_id: {sample.get('goods_id')}, "
  505. f"min_price: {sample.get('min_price')}")
  506. if total_basic_count != len(info_data):
  507. logger.warning(f"警告: 基本价格数据数量({total_basic_count})与饰品记录数量({len(info_data)})不一致")
  508. return detailed_price_list, basic_price_list
  509. # ============================================================
  510. # 主逻辑
  511. # ============================================================
  512. class FloatPriceSyncTask:
  513. """磨损区间价格数据同步任务"""
  514. def __init__(self):
  515. self.db = None
  516. self.api = BuffFloatPriceAPI()
  517. def _init_db(self):
  518. """初始化数据库连接"""
  519. if self.db is None:
  520. self.db = FloatPriceDatabase()
  521. def run(self) -> bool:
  522. """执行同步任务"""
  523. start_time = datetime.now()
  524. logger.info("=" * 60)
  525. logger.info(f"开始执行磨损区间价格数据同步任务: {start_time}")
  526. logger.info("=" * 60)
  527. try:
  528. # 初始化数据库连接
  529. self._init_db()
  530. # Step 1: 清空磨损区间中间表
  531. logger.info("[Step 1/8] 清空buff_goods_float_price_middle表...")
  532. clear_start = time.time()
  533. self.db.clear_table("buff_goods_float_price_middle")
  534. logger.info(f"清空中间表耗时: {time.time() - clear_start:.2f}秒")
  535. # Step 2: 调用BUFF API获取磨损区间数据
  536. logger.info("[Step 2/8] 调用BUFF API获取磨损区间价格数据...")
  537. api_start = time.time()
  538. api_response = self.api.fetch_float_price_data(game="cs2", include_sticker=1)
  539. logger.info(f"API调用耗时: {time.time() - api_start:.2f}秒")
  540. # Step 3: 解析数据
  541. logger.info("[Step 3/8] 解析磨损区间数据...")
  542. parse_start = time.time()
  543. detailed_price_list, basic_price_list = self.api.parse_float_price_data(api_response)
  544. logger.info(f"数据解析耗时: {time.time() - parse_start:.2f}秒")
  545. if not detailed_price_list and not basic_price_list:
  546. logger.warning("未获取到任何价格数据,任务取消")
  547. return False
  548. # Step 4: 插入详细价格数据到中间表(有tag和磨损区间的数据)
  549. logger.info("[Step 4/8] 插入详细价格数据到中间表...")
  550. inserted_detailed_count = self.db.insert_float_price_middle(detailed_price_list)
  551. # Step 5: 清空目标表并更新主表
  552. logger.info("[Step 5/8] 清空buff_goods_float_price表并更新数据...")
  553. copy_start = time.time()
  554. float_inserted = self.db.copy_from_middle_to_float_price()
  555. logger.info(f"详细价格主表插入完成: {float_inserted} 条,耗时: {time.time() - copy_start:.2f}秒")
  556. # Step 6: 清空属性中间表
  557. logger.info("[Step 6/8] 清空buff_goods_float_price_attr_middle表...")
  558. clear_attr_start = time.time()
  559. self.db.clear_table("buff_goods_float_price_attr_middle")
  560. logger.info(f"清空属性中间表耗时: {time.time() - clear_attr_start:.2f}秒")
  561. # Step 7: 插入基本价格数据到中间表
  562. logger.info("[Step 7/8] 插入基本价格数据到中间表...")
  563. inserted_basic_count = self.db.insert_float_price_attr_middle(basic_price_list)
  564. # Step 8: 清空目标表并更新属性主表
  565. logger.info("[Step 8/8] 清空buff_goods_float_price_attr表并更新数据...")
  566. copy_attr_start = time.time()
  567. attr_inserted = self.db.copy_from_middle_to_float_price_attr()
  568. logger.info(f"基本价格主表插入完成: {attr_inserted} 条,耗时: {time.time() - copy_attr_start:.2f}秒")
  569. # 清理中间表
  570. logger.info("[清理] 清空中间表数据...")
  571. cleanup_mid_start = time.time()
  572. self.db.clear_table("buff_goods_float_price_middle")
  573. self.db.clear_table("buff_goods_float_price_attr_middle")
  574. logger.info(f"清理中间表耗时: {time.time() - cleanup_mid_start:.2f}秒")
  575. # 任务完成
  576. end_time = datetime.now()
  577. duration = (end_time - start_time).total_seconds()
  578. logger.info("=" * 60)
  579. logger.info(f"磨损区间价格数据同步任务完成! 总耗时: {duration:.2f}秒")
  580. logger.info(f"同步数据量:")
  581. logger.info(f" 详细价格数据: {inserted_detailed_count} 条(有tag或有磨损)")
  582. logger.info(f" 基本价格数据: {inserted_basic_count} 条(无tag无磨损)")
  583. logger.info(f" 总处理饰品记录: {len(api_response.get('info', []))} 条")
  584. logger.info("=" * 60)
  585. return True
  586. except Exception as e:
  587. if is_database_connection_error(e):
  588. logger.exception("数据库连接异常,本轮跳过,下一轮将重试: %s", e)
  589. return True
  590. logger.exception(f"同步任务失败: {e}")
  591. return False
  592. finally:
  593. if self.db:
  594. self.db.close()
  595. self.db = None
  596. def run_once() -> bool:
  597. """执行一次同步任务"""
  598. task = FloatPriceSyncTask()
  599. return task.run()
  600. if __name__ == "__main__":
  601. sys.exit(0 if run_once() else 1)