|
|
@@ -0,0 +1,478 @@
|
|
|
+#!/usr/bin/env python3
|
|
|
+"""
|
|
|
+按手机号/邮箱导出用户对话记录与能量石消费记录。
|
|
|
+
|
|
|
+输入:用户数据.txt(每行一个手机号或邮箱)
|
|
|
+输出:output/ 目录下,每个用户一个 Excel(两个 sheet:对话记录、能量石消费记录)
|
|
|
+"""
|
|
|
+
|
|
|
+from __future__ import annotations
|
|
|
+
|
|
|
+import argparse
|
|
|
+import json
|
|
|
+import os
|
|
|
+import re
|
|
|
+import sys
|
|
|
+from dataclasses import dataclass
|
|
|
+from datetime import date, datetime
|
|
|
+from decimal import Decimal
|
|
|
+from pathlib import Path
|
|
|
+from typing import Any
|
|
|
+
|
|
|
+import pymysql
|
|
|
+from openpyxl import Workbook
|
|
|
+from openpyxl.styles import Font
|
|
|
+from openpyxl.utils import get_column_letter
|
|
|
+
|
|
|
+
|
|
|
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
|
+DEFAULT_INPUT = Path(__file__).resolve().parent / "用户数据.txt"
|
|
|
+DEFAULT_OUTPUT_DIR = Path(__file__).resolve().parent / "output"
|
|
|
+
|
|
|
+MESSAGE_HEADERS = [
|
|
|
+ "轮次",
|
|
|
+ "角色",
|
|
|
+ "内容",
|
|
|
+ "情感",
|
|
|
+ "Token数",
|
|
|
+ "创建时间",
|
|
|
+ "消息ID",
|
|
|
+ "扩展信息",
|
|
|
+]
|
|
|
+
|
|
|
+CONSUMPTION_HEADERS = [
|
|
|
+ "记录ID",
|
|
|
+ "订单ID",
|
|
|
+ "商品ID",
|
|
|
+ "商品名称",
|
|
|
+ "购买数量",
|
|
|
+ "能量石单价",
|
|
|
+ "水晶石单价",
|
|
|
+ "消耗能量石",
|
|
|
+ "消耗水晶石",
|
|
|
+ "图片名称",
|
|
|
+ "消费时间",
|
|
|
+ "数据来源",
|
|
|
+]
|
|
|
+
|
|
|
+SUMMARY_HEADERS = [
|
|
|
+ "序号",
|
|
|
+ "联系方式",
|
|
|
+ "类型",
|
|
|
+ "是否匹配",
|
|
|
+ "平台用户名",
|
|
|
+ "user_id",
|
|
|
+ "手机号",
|
|
|
+ "邮箱",
|
|
|
+ "对话条数",
|
|
|
+ "消费记录条数",
|
|
|
+ "导出文件名",
|
|
|
+ "备注",
|
|
|
+]
|
|
|
+
|
|
|
+
|
|
|
+@dataclass
|
|
|
+class Contact:
|
|
|
+ raw: str
|
|
|
+ is_email: bool
|
|
|
+
|
|
|
+ @property
|
|
|
+ def norm(self) -> str:
|
|
|
+ value = self.raw.strip()
|
|
|
+ return value.lower() if self.is_email else value
|
|
|
+
|
|
|
+
|
|
|
+def parse_env_file(env_path: Path) -> dict[str, str]:
|
|
|
+ env_map: dict[str, str] = {}
|
|
|
+ if not env_path.exists():
|
|
|
+ return env_map
|
|
|
+
|
|
|
+ for raw in env_path.read_text(encoding="utf-8").splitlines():
|
|
|
+ line = raw.strip()
|
|
|
+ if not line or line.startswith("#"):
|
|
|
+ continue
|
|
|
+ if line.startswith("export "):
|
|
|
+ line = line[len("export ") :].strip()
|
|
|
+ if "=" not in line:
|
|
|
+ continue
|
|
|
+ key, val = line.split("=", 1)
|
|
|
+ key = key.strip()
|
|
|
+ val = val.strip()
|
|
|
+ if not key:
|
|
|
+ continue
|
|
|
+ if len(val) >= 2 and ((val[0] == '"' and val[-1] == '"') or (val[0] == "'" and val[-1] == "'")):
|
|
|
+ val = val[1:-1]
|
|
|
+ env_map[key] = val
|
|
|
+ return env_map
|
|
|
+
|
|
|
+
|
|
|
+def load_db_env(project_root: Path, explicit_env_file: str | None = None) -> dict[str, str]:
|
|
|
+ candidates: list[Path] = []
|
|
|
+ if explicit_env_file:
|
|
|
+ env_path = Path(explicit_env_file)
|
|
|
+ if not env_path.is_absolute():
|
|
|
+ env_path = (project_root / env_path).resolve()
|
|
|
+ candidates.append(env_path)
|
|
|
+ else:
|
|
|
+ candidates.append((project_root / "backend/.env.lyrria").resolve())
|
|
|
+ candidates.append((project_root / "backend/.env").resolve())
|
|
|
+
|
|
|
+ merged: dict[str, str] = {}
|
|
|
+ for env_path in candidates:
|
|
|
+ merged.update(parse_env_file(env_path))
|
|
|
+
|
|
|
+ for key in ("MYSQL_HOST", "MYSQL_PORT", "MYSQL_USER", "MYSQL_PASSWORD", "MYSQL_DATABASE"):
|
|
|
+ if os.getenv(key) is not None:
|
|
|
+ merged[key] = os.getenv(key, "")
|
|
|
+ return merged
|
|
|
+
|
|
|
+
|
|
|
+def get_db_config(env_values: dict[str, str]) -> dict[str, Any]:
|
|
|
+ return {
|
|
|
+ "host": env_values.get("MYSQL_HOST", "127.0.0.1"),
|
|
|
+ "port": int(env_values.get("MYSQL_PORT", "3306")),
|
|
|
+ "user": env_values.get("MYSQL_USER", "root"),
|
|
|
+ "password": env_values.get("MYSQL_PASSWORD", ""),
|
|
|
+ "database": env_values.get("MYSQL_DATABASE", "Lyrria"),
|
|
|
+ "charset": "utf8mb4",
|
|
|
+ "cursorclass": pymysql.cursors.DictCursor,
|
|
|
+ "autocommit": True,
|
|
|
+ "connect_timeout": int(env_values.get("MYSQL_CONNECT_TIMEOUT", "10")),
|
|
|
+ "read_timeout": int(env_values.get("MYSQL_READ_TIMEOUT", "60")),
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def parse_contacts(input_path: Path) -> list[Contact]:
|
|
|
+ contacts: list[Contact] = []
|
|
|
+ seen: set[str] = set()
|
|
|
+
|
|
|
+ for raw in input_path.read_text(encoding="utf-8").splitlines():
|
|
|
+ line = raw.strip()
|
|
|
+ if not line:
|
|
|
+ continue
|
|
|
+ is_email = "@" in line
|
|
|
+ norm = line.lower() if is_email else line
|
|
|
+ if norm in seen:
|
|
|
+ continue
|
|
|
+ seen.add(norm)
|
|
|
+ contacts.append(Contact(raw=line, is_email=is_email))
|
|
|
+ return contacts
|
|
|
+
|
|
|
+
|
|
|
+def fetch_users_by_contacts(cur, contacts: list[Contact]) -> dict[str, dict[str, Any]]:
|
|
|
+ phones = sorted({c.norm for c in contacts if not c.is_email})
|
|
|
+ emails = sorted({c.norm for c in contacts if c.is_email})
|
|
|
+ matched: dict[str, dict[str, Any]] = {}
|
|
|
+
|
|
|
+ if phones:
|
|
|
+ placeholders = ", ".join(["%s"] * len(phones))
|
|
|
+ sql = (
|
|
|
+ "SELECT id, user_id, username, phone, email, area_code, user_type, "
|
|
|
+ "registered_at, total_messages, total_rounds, conversation_energy_stone_total, "
|
|
|
+ "crystal_stone_total, created_at "
|
|
|
+ f"FROM users WHERE phone IN ({placeholders})"
|
|
|
+ )
|
|
|
+ cur.execute(sql, phones)
|
|
|
+ for row in cur.fetchall():
|
|
|
+ phone = row.get("phone")
|
|
|
+ if phone:
|
|
|
+ matched[str(phone)] = row
|
|
|
+
|
|
|
+ if emails:
|
|
|
+ placeholders = ", ".join(["%s"] * len(emails))
|
|
|
+ sql = (
|
|
|
+ "SELECT id, user_id, username, phone, email, area_code, user_type, "
|
|
|
+ "registered_at, total_messages, total_rounds, conversation_energy_stone_total, "
|
|
|
+ "crystal_stone_total, created_at "
|
|
|
+ f"FROM users WHERE LOWER(email) IN ({placeholders})"
|
|
|
+ )
|
|
|
+ cur.execute(sql, emails)
|
|
|
+ for row in cur.fetchall():
|
|
|
+ email = row.get("email")
|
|
|
+ if email:
|
|
|
+ matched[email.strip().lower()] = row
|
|
|
+
|
|
|
+ return matched
|
|
|
+
|
|
|
+
|
|
|
+def fetch_messages(cur, user_id: str) -> list[dict[str, Any]]:
|
|
|
+ sql = (
|
|
|
+ "SELECT message_id, round_number, role, content, emotion, token_count, "
|
|
|
+ "message_metadata, created_at "
|
|
|
+ "FROM messages "
|
|
|
+ "WHERE user_id = %s "
|
|
|
+ "ORDER BY round_number ASC, id ASC"
|
|
|
+ )
|
|
|
+ cur.execute(sql, (user_id,))
|
|
|
+ return list(cur.fetchall())
|
|
|
+
|
|
|
+
|
|
|
+def _user_match_clause(user_uuid: str, user_pk: int) -> tuple[str, list[Any]]:
|
|
|
+ return (
|
|
|
+ "(CAST(apl.user_id AS CHAR) = %s OR apl.user_id = %s)",
|
|
|
+ [user_uuid, user_pk],
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+def fetch_consumption_logs(cur, user_uuid: str, user_pk: int) -> list[dict[str, Any]]:
|
|
|
+ rows: list[dict[str, Any]] = []
|
|
|
+
|
|
|
+ where_sql, params = _user_match_clause(user_uuid, user_pk)
|
|
|
+ try:
|
|
|
+ sql = (
|
|
|
+ "SELECT apl.id, NULL AS order_id, apl.item_id, apl.quantity, "
|
|
|
+ "apl.unit_price_energy, apl.unit_price_crystal, apl.cost_energy, apl.cost_crystal, "
|
|
|
+ "apl.item_name_snapshot, apl.image_name_snapshot, apl.created_at, "
|
|
|
+ "'asset_purchase_logs' AS source_table "
|
|
|
+ "FROM asset_purchase_logs apl "
|
|
|
+ f"WHERE {where_sql} "
|
|
|
+ "ORDER BY apl.created_at ASC, apl.id ASC"
|
|
|
+ )
|
|
|
+ cur.execute(sql, params)
|
|
|
+ rows.extend(cur.fetchall())
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+
|
|
|
+ if rows:
|
|
|
+ return rows
|
|
|
+
|
|
|
+ try:
|
|
|
+ sql = (
|
|
|
+ "SELECT poi.id, po.id AS order_id, poi.item_id, poi.quantity, "
|
|
|
+ "poi.unit_price_energy, poi.unit_price_crystal, poi.cost_energy, poi.cost_crystal, "
|
|
|
+ "poi.item_name_snapshot, poi.image_name_snapshot, "
|
|
|
+ "COALESCE(po.completed_at, poi.created_at) AS created_at, "
|
|
|
+ "'purchase_order_items' AS source_table "
|
|
|
+ "FROM purchase_order_items poi "
|
|
|
+ "JOIN purchase_orders po ON poi.order_id = po.id "
|
|
|
+ "WHERE po.status = 'completed' "
|
|
|
+ "AND (CAST(po.user_id AS CHAR) = %s OR po.user_id = %s) "
|
|
|
+ "ORDER BY created_at ASC, poi.id ASC"
|
|
|
+ )
|
|
|
+ cur.execute(sql, [user_uuid, user_pk])
|
|
|
+ rows.extend(cur.fetchall())
|
|
|
+ except Exception:
|
|
|
+ pass
|
|
|
+
|
|
|
+ return rows
|
|
|
+
|
|
|
+
|
|
|
+def normalize_cell(value: Any) -> Any:
|
|
|
+ if value is None:
|
|
|
+ return ""
|
|
|
+ if isinstance(value, datetime):
|
|
|
+ return value.strftime("%Y-%m-%d %H:%M:%S")
|
|
|
+ if isinstance(value, date):
|
|
|
+ return value.strftime("%Y-%m-%d")
|
|
|
+ if isinstance(value, Decimal):
|
|
|
+ return float(value)
|
|
|
+ if isinstance(value, (dict, list)):
|
|
|
+ return json.dumps(value, ensure_ascii=False)
|
|
|
+ return value
|
|
|
+
|
|
|
+
|
|
|
+def autosize_columns(ws, max_width: int = 60) -> None:
|
|
|
+ for col_idx, column_cells in enumerate(ws.columns, start=1):
|
|
|
+ max_len = 0
|
|
|
+ for cell in column_cells:
|
|
|
+ text = "" if cell.value is None else str(cell.value)
|
|
|
+ max_len = max(max_len, len(text))
|
|
|
+ width = min(max(max_len + 2, 10), max_width)
|
|
|
+ ws.column_dimensions[get_column_letter(col_idx)].width = width
|
|
|
+
|
|
|
+
|
|
|
+def write_sheet_header(ws, headers: list[str]) -> None:
|
|
|
+ ws.append(headers)
|
|
|
+ for cell in ws[1]:
|
|
|
+ cell.font = Font(bold=True)
|
|
|
+
|
|
|
+
|
|
|
+def sanitize_filename(name: str) -> str:
|
|
|
+ cleaned = re.sub(r'[\\/:*?"<>|]+', "_", name.strip())
|
|
|
+ cleaned = re.sub(r"\s+", "_", cleaned)
|
|
|
+ return cleaned[:120] or "unknown"
|
|
|
+
|
|
|
+
|
|
|
+def build_output_filename(contact: Contact, user_row: dict[str, Any]) -> str:
|
|
|
+ username = sanitize_filename(str(user_row.get("username") or "user"))
|
|
|
+ contact_part = sanitize_filename(contact.raw)
|
|
|
+ return f"{contact_part}_{username}.xlsx"
|
|
|
+
|
|
|
+
|
|
|
+def export_user_workbook(
|
|
|
+ output_path: Path,
|
|
|
+ user_row: dict[str, Any],
|
|
|
+ messages: list[dict[str, Any]],
|
|
|
+ consumptions: list[dict[str, Any]],
|
|
|
+) -> None:
|
|
|
+ wb = Workbook()
|
|
|
+
|
|
|
+ ws_messages = wb.active
|
|
|
+ ws_messages.title = "对话记录"
|
|
|
+ write_sheet_header(ws_messages, MESSAGE_HEADERS)
|
|
|
+ for row in messages:
|
|
|
+ ws_messages.append(
|
|
|
+ [
|
|
|
+ normalize_cell(row.get("round_number")),
|
|
|
+ normalize_cell(row.get("role")),
|
|
|
+ normalize_cell(row.get("content")),
|
|
|
+ normalize_cell(row.get("emotion")),
|
|
|
+ normalize_cell(row.get("token_count")),
|
|
|
+ normalize_cell(row.get("created_at")),
|
|
|
+ normalize_cell(row.get("message_id")),
|
|
|
+ normalize_cell(row.get("message_metadata")),
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ autosize_columns(ws_messages)
|
|
|
+
|
|
|
+ ws_consume = wb.create_sheet("能量石消费记录")
|
|
|
+ write_sheet_header(ws_consume, CONSUMPTION_HEADERS)
|
|
|
+ for row in consumptions:
|
|
|
+ ws_consume.append(
|
|
|
+ [
|
|
|
+ normalize_cell(row.get("id")),
|
|
|
+ normalize_cell(row.get("order_id")),
|
|
|
+ normalize_cell(row.get("item_id")),
|
|
|
+ normalize_cell(row.get("item_name_snapshot")),
|
|
|
+ normalize_cell(row.get("quantity")),
|
|
|
+ normalize_cell(row.get("unit_price_energy")),
|
|
|
+ normalize_cell(row.get("unit_price_crystal")),
|
|
|
+ normalize_cell(row.get("cost_energy")),
|
|
|
+ normalize_cell(row.get("cost_crystal")),
|
|
|
+ normalize_cell(row.get("image_name_snapshot")),
|
|
|
+ normalize_cell(row.get("created_at")),
|
|
|
+ normalize_cell(row.get("source_table")),
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ autosize_columns(ws_consume)
|
|
|
+
|
|
|
+ output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
+ wb.save(output_path)
|
|
|
+
|
|
|
+
|
|
|
+def export_summary_workbook(output_dir: Path, summary_rows: list[list[Any]]) -> Path:
|
|
|
+ summary_path = output_dir / "_导出汇总.xlsx"
|
|
|
+ wb = Workbook()
|
|
|
+ ws = wb.active
|
|
|
+ ws.title = "导出汇总"
|
|
|
+ write_sheet_header(ws, SUMMARY_HEADERS)
|
|
|
+ for row in summary_rows:
|
|
|
+ ws.append(row)
|
|
|
+ autosize_columns(ws)
|
|
|
+ wb.save(summary_path)
|
|
|
+ return summary_path
|
|
|
+
|
|
|
+
|
|
|
+def main() -> int:
|
|
|
+ parser = argparse.ArgumentParser(description="按手机号/邮箱导出用户对话与能量石消费记录")
|
|
|
+ parser.add_argument(
|
|
|
+ "--input",
|
|
|
+ default=str(DEFAULT_INPUT),
|
|
|
+ help="用户名单文件路径(默认:数据提取/用户数据.txt)",
|
|
|
+ )
|
|
|
+ parser.add_argument(
|
|
|
+ "--output-dir",
|
|
|
+ default=str(DEFAULT_OUTPUT_DIR),
|
|
|
+ help="Excel 输出目录(默认:数据提取/output)",
|
|
|
+ )
|
|
|
+ parser.add_argument(
|
|
|
+ "--env-file",
|
|
|
+ default="",
|
|
|
+ help="可选:指定 env 文件路径(默认 backend/.env.lyrria)",
|
|
|
+ )
|
|
|
+ args = parser.parse_args()
|
|
|
+
|
|
|
+ input_path = Path(args.input).resolve()
|
|
|
+ output_dir = Path(args.output_dir).resolve()
|
|
|
+
|
|
|
+ if not input_path.exists():
|
|
|
+ print(f"输入文件不存在: {input_path}", file=sys.stderr)
|
|
|
+ return 1
|
|
|
+
|
|
|
+ contacts = parse_contacts(input_path)
|
|
|
+ if not contacts:
|
|
|
+ print("输入文件中没有有效的手机号或邮箱", file=sys.stderr)
|
|
|
+ return 1
|
|
|
+
|
|
|
+ env_values = load_db_env(PROJECT_ROOT, args.env_file or None)
|
|
|
+ db_conf = get_db_config(env_values)
|
|
|
+
|
|
|
+ try:
|
|
|
+ conn = pymysql.connect(**db_conf)
|
|
|
+ except Exception as exc:
|
|
|
+ print(f"数据库连接失败: {exc}", file=sys.stderr)
|
|
|
+ return 1
|
|
|
+
|
|
|
+ summary_rows: list[list[Any]] = []
|
|
|
+ exported_count = 0
|
|
|
+ matched_count = 0
|
|
|
+
|
|
|
+ try:
|
|
|
+ with conn.cursor() as cur:
|
|
|
+ user_map = fetch_users_by_contacts(cur, contacts)
|
|
|
+
|
|
|
+ for idx, contact in enumerate(contacts, start=1):
|
|
|
+ user_row = user_map.get(contact.norm)
|
|
|
+ if not user_row:
|
|
|
+ summary_rows.append(
|
|
|
+ [
|
|
|
+ idx,
|
|
|
+ contact.raw,
|
|
|
+ "邮箱" if contact.is_email else "手机号",
|
|
|
+ "否",
|
|
|
+ "",
|
|
|
+ "",
|
|
|
+ "",
|
|
|
+ "",
|
|
|
+ 0,
|
|
|
+ 0,
|
|
|
+ "",
|
|
|
+ "未找到对应用户",
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ continue
|
|
|
+
|
|
|
+ matched_count += 1
|
|
|
+ user_uuid = str(user_row["user_id"])
|
|
|
+ user_pk = int(user_row["id"])
|
|
|
+
|
|
|
+ messages = fetch_messages(cur, user_uuid)
|
|
|
+ consumptions = fetch_consumption_logs(cur, user_uuid, user_pk)
|
|
|
+
|
|
|
+ filename = build_output_filename(contact, user_row)
|
|
|
+ output_path = output_dir / filename
|
|
|
+ export_user_workbook(output_path, user_row, messages, consumptions)
|
|
|
+ exported_count += 1
|
|
|
+
|
|
|
+ summary_rows.append(
|
|
|
+ [
|
|
|
+ idx,
|
|
|
+ contact.raw,
|
|
|
+ "邮箱" if contact.is_email else "手机号",
|
|
|
+ "是",
|
|
|
+ user_row.get("username") or "",
|
|
|
+ user_uuid,
|
|
|
+ user_row.get("phone") or "",
|
|
|
+ user_row.get("email") or "",
|
|
|
+ len(messages),
|
|
|
+ len(consumptions),
|
|
|
+ filename,
|
|
|
+ "",
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ summary_path = export_summary_workbook(output_dir, summary_rows)
|
|
|
+
|
|
|
+ print(f"输入联系方式(去重后): {len(contacts)}")
|
|
|
+ print(f"匹配成功: {matched_count}")
|
|
|
+ print(f"未匹配: {len(contacts) - matched_count}")
|
|
|
+ print(f"已导出 Excel: {exported_count}")
|
|
|
+ print(f"输出目录: {output_dir}")
|
|
|
+ print(f"汇总文件: {summary_path}")
|
|
|
+ return 0
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ raise SystemExit(main())
|