migrate_crm_customer.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655
  1. #!/usr/bin/env python3
  2. """将 sx 库 CRM 客户相关表迁入 shanxia2.0。
  3. 顺序:crm_customer → crm_contact → crm_customer_follow → crm_customer_pool
  4. 证照字段 OSS ID → 目标 sys_oss 相对 URL;用户外键经 user_mapping 转换。
  5. 用法:
  6. cd backend
  7. venv/bin/python scripts/migrate_crm_customer.py --dry-run --env-file .env.production
  8. venv/bin/python scripts/migrate_crm_customer.py --env-file .env.production
  9. """
  10. from __future__ import annotations
  11. import argparse
  12. import csv
  13. import os
  14. import sys
  15. from dataclasses import dataclass, field
  16. from datetime import datetime
  17. from decimal import Decimal
  18. from pathlib import Path
  19. from typing import Any, Dict, List, Optional, Tuple
  20. from urllib.parse import urlparse, urlunparse
  21. from sqlalchemy import create_engine, text
  22. from sqlalchemy.engine import Engine
  23. BACKEND_ROOT = Path(__file__).resolve().parents[1]
  24. MIGRATION_DIR = BACKEND_ROOT / "scripts" / "migration"
  25. if str(BACKEND_ROOT) not in sys.path:
  26. sys.path.insert(0, str(BACKEND_ROOT))
  27. if str(MIGRATION_DIR) not in sys.path:
  28. sys.path.insert(0, str(MIGRATION_DIR))
  29. from minio_url import resolve_oss_id_csv # noqa: E402
  30. DEFAULT_TENANT_NO = "shanxia2.0"
  31. CRM_ADMIN_SOURCE_USER_ID = 1
  32. CUSTOMER_TYPE_MAP = {
  33. "company": "company",
  34. "person": "person",
  35. "未分类": None,
  36. "a类国央企单位": "A类国央企业",
  37. "A类国央企单位": "A类国央企业",
  38. "A类国央企业": "A类国央企业",
  39. "b类股份制企业": "B类股份制企业",
  40. "B类股份制企业": "B类股份制企业",
  41. "c类私营单位": "C类私营企业",
  42. "C类私营企业": "C类私营企业",
  43. "d外商投资企业": "D外商投资企业",
  44. "D外商投资企业": "D外商投资企业",
  45. "e类国资委企业": "5",
  46. "5": "5",
  47. }
  48. FOLLOW_TYPE_MAP = {
  49. "电话": "phone",
  50. "phone": "phone",
  51. "微信": "wechat",
  52. "wechat": "wechat",
  53. "邮件": "email",
  54. "email": "email",
  55. "拜访": "visit",
  56. "visit": "visit",
  57. "其他": "other",
  58. "other": "other",
  59. }
  60. MIGRATION_STEPS = (
  61. ("crm_customer", "customer"),
  62. ("crm_contact", "contact"),
  63. ("crm_customer_follow", "follow"),
  64. ("crm_customer_pool", "pool"),
  65. )
  66. @dataclass
  67. class StepStats:
  68. source_count: int = 0
  69. inserted: int = 0
  70. updated: int = 0
  71. skipped: int = 0
  72. @dataclass
  73. class MigrationStats:
  74. steps: Dict[str, StepStats] = field(default_factory=dict)
  75. warnings: List[str] = field(default_factory=list)
  76. def step(self, name: str) -> StepStats:
  77. if name not in self.steps:
  78. self.steps[name] = StepStats()
  79. return self.steps[name]
  80. def legacy_id_to_str(legacy_id: Any) -> Optional[str]:
  81. if legacy_id is None or str(legacy_id).strip() == "":
  82. return None
  83. try:
  84. return str(int(legacy_id))
  85. except (TypeError, ValueError):
  86. return str(legacy_id).strip()
  87. def derive_source_url(target_url: str, source_db: str = "sx") -> str:
  88. return urlunparse(urlparse(target_url)._replace(path=f"/{source_db}"))
  89. def load_settings_env(env_file: Optional[str]) -> None:
  90. if env_file:
  91. os.environ["ENV_FILE"] = env_file
  92. from dotenv import load_dotenv
  93. candidates: List[Path] = []
  94. if env_file:
  95. p = Path(env_file)
  96. candidates.append(p if p.is_absolute() else BACKEND_ROOT / p)
  97. candidates.extend([BACKEND_ROOT / ".env.production", BACKEND_ROOT / ".env"])
  98. for path in candidates:
  99. if path.is_file():
  100. load_dotenv(path, override=True)
  101. break
  102. def load_runtime_settings():
  103. from app.core.config import Settings
  104. return Settings()
  105. def load_user_mapping(path: Path) -> Tuple[Dict[int, str], Optional[str]]:
  106. crm_map: Dict[int, str] = {}
  107. admin_id: Optional[str] = None
  108. with path.open(encoding="utf-8-sig", newline="") as f:
  109. for row in csv.DictReader(f):
  110. new_id = (row.get("new_user_id") or "").strip()
  111. user_no = (row.get("user_no") or "").strip()
  112. crm_raw = (row.get("crm_user_id") or "").strip()
  113. if user_no == "admin" and new_id:
  114. admin_id = new_id
  115. if crm_raw and new_id:
  116. try:
  117. crm_map[int(crm_raw)] = new_id
  118. except ValueError:
  119. pass
  120. return crm_map, admin_id
  121. def fetch_portal_admin_id(target_engine: Engine) -> Optional[str]:
  122. with target_engine.connect() as conn:
  123. row = conn.execute(
  124. text("SELECT id FROM sys_user WHERE user_no='admin' AND del_flag=0 LIMIT 1")
  125. ).first()
  126. return str(row[0]) if row and row[0] else None
  127. def resolve_portal_user_id(
  128. source_user_id: Any,
  129. crm_user_map: Dict[int, str],
  130. admin_id: Optional[str],
  131. stats: MigrationStats,
  132. label: str,
  133. *,
  134. required: bool = False,
  135. ) -> Optional[str]:
  136. if source_user_id is None or str(source_user_id).strip() == "":
  137. return None
  138. try:
  139. uid = int(source_user_id)
  140. except (TypeError, ValueError):
  141. stats.warnings.append(f"{label}: 非法用户 id {source_user_id!r}")
  142. return None
  143. if uid == CRM_ADMIN_SOURCE_USER_ID:
  144. return admin_id
  145. mapped = crm_user_map.get(uid)
  146. if not mapped:
  147. msg = f"{label}: crm_user_id={uid} 无映射"
  148. if required:
  149. stats.warnings.append(msg)
  150. return mapped
  151. def load_oss_id_to_url(target_engine: Engine, tenant_no: str) -> Dict[str, str]:
  152. with target_engine.connect() as conn:
  153. rows = conn.execute(
  154. text("SELECT id, url FROM sys_oss WHERE tenant_no=:t AND del_flag=0"),
  155. {"t": tenant_no},
  156. ).mappings().all()
  157. return {str(r["id"]): (r["url"] or "").strip() for r in rows if r["id"]}
  158. def normalize_customer_type(raw: Any, stats: MigrationStats, label: str) -> Optional[str]:
  159. value = (str(raw).strip() if raw is not None else "") or ""
  160. if not value:
  161. return None
  162. if value in CUSTOMER_TYPE_MAP:
  163. mapped = CUSTOMER_TYPE_MAP[value]
  164. if mapped is None:
  165. stats.warnings.append(f"{label}: customer_type={value!r} 已置空")
  166. return mapped
  167. lowered = value.lower()
  168. for key, mapped in CUSTOMER_TYPE_MAP.items():
  169. if key.lower() == lowered:
  170. return mapped
  171. stats.warnings.append(f"{label}: customer_type={value!r} 未识别,已置空")
  172. return None
  173. def normalize_follow_type(raw: Any) -> str:
  174. value = (str(raw).strip() if raw is not None else "") or "other"
  175. return FOLLOW_TYPE_MAP.get(value, FOLLOW_TYPE_MAP.get(value.lower(), "other"))
  176. def _dec(value: Any) -> Optional[Decimal]:
  177. if value is None or value == "":
  178. return None
  179. return Decimal(str(value))
  180. def _str_or_none(value: Any, max_len: int = 0) -> Optional[str]:
  181. if value is None:
  182. return None
  183. s = str(value).strip()
  184. if not s:
  185. return None
  186. return s[:max_len] if max_len else s
  187. def fetch_source_rows(engine: Engine, table: str, include_deleted: bool) -> List[dict]:
  188. where = "" if include_deleted else "WHERE del_flag = 0"
  189. sql = f"SELECT * FROM `{table}` {where} ORDER BY id ASC"
  190. with engine.connect() as conn:
  191. return [dict(r._mapping) for r in conn.execute(text(sql))]
  192. def target_row_exists(conn, table: str, row_id: str) -> bool:
  193. return int(conn.execute(text(f"SELECT COUNT(*) FROM `{table}` WHERE id=:id"), {"id": row_id}).scalar_one()) > 0
  194. def transform_customer(
  195. row: dict,
  196. tenant_no: str,
  197. crm_user_map: Dict[int, str],
  198. admin_id: Optional[str],
  199. oss_map: Dict[str, str],
  200. stats: MigrationStats,
  201. ) -> dict:
  202. rid = legacy_id_to_str(row["id"])
  203. label = f"customer#{rid}"
  204. def map_user(field_val: Any) -> Optional[str]:
  205. return resolve_portal_user_id(field_val, crm_user_map, admin_id, stats, f"{label} {field_val}")
  206. return {
  207. "id": rid,
  208. "customer_no": (row.get("customer_no") or "")[:50],
  209. "customer_name": (row.get("customer_name") or "")[:200],
  210. "short_name": _str_or_none(row.get("short_name"), 100),
  211. "customer_type": normalize_customer_type(row.get("customer_type"), stats, label),
  212. "mobile": _str_or_none(row.get("mobile"), 20),
  213. "telephone": _str_or_none(row.get("telephone"), 30),
  214. "email": _str_or_none(row.get("email"), 100),
  215. "website": _str_or_none(row.get("website"), 500),
  216. "level": (row.get("level") or "normal")[:20],
  217. "source": _str_or_none(row.get("source"), 20),
  218. "industry": _str_or_none(row.get("industry"), 50),
  219. "park_area": _str_or_none(row.get("park_area"), 100),
  220. "payment_terms": _str_or_none(row.get("payment_terms"), 500),
  221. "scale": _str_or_none(row.get("scale"), 20),
  222. "province": _str_or_none(row.get("province"), 50),
  223. "city": _str_or_none(row.get("city"), 50),
  224. "district": _str_or_none(row.get("district"), 50),
  225. "address": _str_or_none(row.get("address"), 255),
  226. "status": (row.get("status") or "normal")[:20],
  227. "deal_status": (row.get("deal_status") or "not_deal")[:20],
  228. "owner_user_id": map_user(row.get("owner_user_id")),
  229. "owner_user_name": _str_or_none(row.get("owner_user_name"), 50),
  230. "last_time": row.get("last_time"),
  231. "next_time": row.get("next_time"),
  232. "follow_count": int(row.get("follow_count") or 0),
  233. "deal_time": row.get("deal_time"),
  234. "deal_amount": _dec(row.get("deal_amount")) or Decimal("0.00"),
  235. "contact_count": int(row.get("contact_count") or 0),
  236. "opportunity_count": int(row.get("opportunity_count") or 0),
  237. "contract_count": int(row.get("contract_count") or 0),
  238. "into_pool_time": row.get("into_pool_time"),
  239. "into_pool_by": map_user(row.get("into_pool_by")),
  240. "into_pool_reason": _str_or_none(row.get("into_pool_reason"), 500),
  241. "claim_time": row.get("claim_time"),
  242. "assign_time": row.get("assign_time"),
  243. "assign_by": map_user(row.get("assign_by")),
  244. "protect_time": row.get("protect_time"),
  245. "business_license_img": resolve_oss_id_csv(row.get("business_license_images"), oss_map) or None,
  246. "business_permit_images": resolve_oss_id_csv(row.get("business_permit_images"), oss_map) or None,
  247. "other_cert_images": resolve_oss_id_csv(row.get("other_cert_images"), oss_map) or None,
  248. "clue_id": legacy_id_to_str(row.get("clue_id")),
  249. "tenant_no": tenant_no,
  250. "order_num": 0,
  251. "remark": _str_or_none(row.get("remark"), 512),
  252. "create_by": map_user(row.get("create_by")),
  253. "create_dept_id": "",
  254. "create_time": row.get("create_time") or datetime.now(),
  255. "update_by": map_user(row.get("update_by")),
  256. "update_time": row.get("update_time") or row.get("create_time") or datetime.now(),
  257. "del_flag": int(row.get("del_flag") or 0),
  258. }
  259. def transform_contact(
  260. row: dict,
  261. tenant_no: str,
  262. crm_user_map: Dict[int, str],
  263. admin_id: Optional[str],
  264. stats: MigrationStats,
  265. ) -> dict:
  266. rid = legacy_id_to_str(row["id"])
  267. label = f"contact#{rid}"
  268. def map_user(field_val: Any) -> Optional[str]:
  269. return resolve_portal_user_id(field_val, crm_user_map, admin_id, stats, label)
  270. return {
  271. "id": rid,
  272. "customer_id": legacy_id_to_str(row.get("customer_id")),
  273. "customer_name": _str_or_none(row.get("customer_name"), 200),
  274. "contact_name": (row.get("contact_name") or "")[:100],
  275. "gender": _str_or_none(row.get("gender"), 10),
  276. "mobile": _str_or_none(row.get("mobile"), 20),
  277. "telephone": _str_or_none(row.get("telephone"), 30),
  278. "email": _str_or_none(row.get("email"), 100),
  279. "wechat": _str_or_none(row.get("wechat"), 100),
  280. "qq": _str_or_none(row.get("qq"), 20),
  281. "department": _str_or_none(row.get("department"), 100),
  282. "position": _str_or_none(row.get("position"), 50),
  283. "role": _str_or_none(row.get("role"), 20),
  284. "birthday": row.get("birthday"),
  285. "address": _str_or_none(row.get("address"), 255),
  286. "is_key": int(row.get("is_key") or 0),
  287. "is_primary": int(row.get("is_primary") or 0),
  288. "status": (row.get("status") or "active")[:20],
  289. "tenant_no": tenant_no,
  290. "order_num": 0,
  291. "remark": _str_or_none(row.get("remark"), 512),
  292. "create_by": map_user(row.get("create_by")),
  293. "create_dept_id": "",
  294. "create_time": row.get("create_time") or datetime.now(),
  295. "update_by": map_user(row.get("update_by")),
  296. "update_time": row.get("update_time") or row.get("create_time") or datetime.now(),
  297. "del_flag": int(row.get("del_flag") or 0),
  298. }
  299. def transform_follow(
  300. row: dict,
  301. tenant_no: str,
  302. crm_user_map: Dict[int, str],
  303. admin_id: Optional[str],
  304. oss_map: Dict[str, str],
  305. stats: MigrationStats,
  306. ) -> dict:
  307. rid = legacy_id_to_str(row["id"])
  308. label = f"follow#{rid}"
  309. owner_id = resolve_portal_user_id(row.get("create_by"), crm_user_map, admin_id, stats, label)
  310. attachment = resolve_oss_id_csv(row.get("attachments"), oss_map) or None
  311. return {
  312. "id": rid,
  313. "customer_id": legacy_id_to_str(row.get("customer_id")),
  314. "customer_name": _str_or_none(row.get("customer_name"), 200),
  315. "contact_id": legacy_id_to_str(row.get("contact_id")),
  316. "contact_name": _str_or_none(row.get("contact_name"), 100),
  317. "follow_type": normalize_follow_type(row.get("follow_type")),
  318. "follow_content": row.get("follow_content") or "",
  319. "follow_time": row.get("create_time"),
  320. "next_time": row.get("next_time"),
  321. "attachment": attachment,
  322. "owner_user_id": owner_id,
  323. "owner_user_name": _str_or_none(row.get("create_by_name"), 50),
  324. "tenant_no": tenant_no,
  325. "order_num": 0,
  326. "remark": _str_or_none(row.get("remark"), 512),
  327. "create_by": owner_id,
  328. "create_dept_id": "",
  329. "create_time": row.get("create_time") or datetime.now(),
  330. "update_by": resolve_portal_user_id(row.get("update_by"), crm_user_map, admin_id, stats, label),
  331. "update_time": row.get("update_time") or row.get("create_time") or datetime.now(),
  332. "del_flag": int(row.get("del_flag") or 0),
  333. }
  334. def transform_pool(
  335. row: dict,
  336. tenant_no: str,
  337. crm_user_map: Dict[int, str],
  338. admin_id: Optional[str],
  339. stats: MigrationStats,
  340. ) -> dict:
  341. rid = legacy_id_to_str(row["id"])
  342. label = f"pool#{rid}"
  343. def map_user(field_val: Any) -> Optional[str]:
  344. return resolve_portal_user_id(field_val, crm_user_map, admin_id, stats, label)
  345. create_by = map_user(row.get("create_by"))
  346. return {
  347. "id": rid,
  348. "customer_id": legacy_id_to_str(row.get("customer_id")),
  349. "customer_name": _str_or_none(row.get("customer_name"), 200),
  350. "operation_type": (row.get("operation_type") or "")[:20],
  351. "from_user_id": map_user(row.get("from_user_id")),
  352. "from_user_name": _str_or_none(row.get("from_user_name"), 50),
  353. "to_user_id": map_user(row.get("to_user_id")),
  354. "to_user_name": _str_or_none(row.get("to_user_name"), 50),
  355. "reason": _str_or_none(row.get("reason"), 500),
  356. "tenant_no": tenant_no,
  357. "order_num": 0,
  358. "remark": _str_or_none(row.get("remark"), 500),
  359. "create_by": create_by,
  360. "create_by_name": _str_or_none(row.get("create_by_name"), 50),
  361. "create_dept_id": "",
  362. "create_time": row.get("create_time") or datetime.now(),
  363. "update_by": map_user(row.get("update_by")),
  364. "update_time": row.get("update_time") or row.get("create_time") or datetime.now(),
  365. "del_flag": int(row.get("del_flag") or 0),
  366. }
  367. INSERT_CUSTOMER_SQL = text(
  368. """
  369. INSERT INTO crm_customer (
  370. id, customer_no, customer_name, short_name, customer_type, mobile, telephone, email, website,
  371. level, source, industry, park_area, payment_terms, scale, province, city, district, address,
  372. status, deal_status, owner_user_id, owner_user_name, last_time, next_time, follow_count,
  373. deal_time, deal_amount, contact_count, opportunity_count, contract_count, into_pool_time,
  374. into_pool_by, into_pool_reason, claim_time, assign_time, assign_by, protect_time,
  375. business_license_img, business_permit_images, other_cert_images, clue_id,
  376. tenant_no, order_num, remark, create_by, create_dept_id, create_time, update_by, update_time, del_flag
  377. ) VALUES (
  378. :id, :customer_no, :customer_name, :short_name, :customer_type, :mobile, :telephone, :email, :website,
  379. :level, :source, :industry, :park_area, :payment_terms, :scale, :province, :city, :district, :address,
  380. :status, :deal_status, :owner_user_id, :owner_user_name, :last_time, :next_time, :follow_count,
  381. :deal_time, :deal_amount, :contact_count, :opportunity_count, :contract_count, :into_pool_time,
  382. :into_pool_by, :into_pool_reason, :claim_time, :assign_time, :assign_by, :protect_time,
  383. :business_license_img, :business_permit_images, :other_cert_images, :clue_id,
  384. :tenant_no, :order_num, :remark, :create_by, :create_dept_id, :create_time, :update_by, :update_time, :del_flag
  385. )
  386. """
  387. )
  388. INSERT_CONTACT_SQL = text(
  389. """
  390. INSERT INTO crm_contact (
  391. id, customer_id, customer_name, contact_name, gender, mobile, telephone, email, wechat, qq,
  392. department, position, role, birthday, address, is_key, is_primary, status,
  393. tenant_no, order_num, remark, create_by, create_dept_id, create_time, update_by, update_time, del_flag
  394. ) VALUES (
  395. :id, :customer_id, :customer_name, :contact_name, :gender, :mobile, :telephone, :email, :wechat, :qq,
  396. :department, :position, :role, :birthday, :address, :is_key, :is_primary, :status,
  397. :tenant_no, :order_num, :remark, :create_by, :create_dept_id, :create_time, :update_by, :update_time, :del_flag
  398. )
  399. """
  400. )
  401. INSERT_FOLLOW_SQL = text(
  402. """
  403. INSERT INTO crm_customer_follow (
  404. id, customer_id, customer_name, contact_id, contact_name, follow_type, follow_content,
  405. follow_time, next_time, attachment, owner_user_id, owner_user_name,
  406. tenant_no, order_num, remark, create_by, create_dept_id, create_time, update_by, update_time, del_flag
  407. ) VALUES (
  408. :id, :customer_id, :customer_name, :contact_id, :contact_name, :follow_type, :follow_content,
  409. :follow_time, :next_time, :attachment, :owner_user_id, :owner_user_name,
  410. :tenant_no, :order_num, :remark, :create_by, :create_dept_id, :create_time, :update_by, :update_time, :del_flag
  411. )
  412. """
  413. )
  414. INSERT_POOL_SQL = text(
  415. """
  416. INSERT INTO crm_customer_pool (
  417. id, customer_id, customer_name, operation_type, from_user_id, from_user_name, to_user_id, to_user_name,
  418. reason, tenant_no, order_num, remark, create_by, create_by_name, create_dept_id,
  419. create_time, update_by, update_time, del_flag
  420. ) VALUES (
  421. :id, :customer_id, :customer_name, :operation_type, :from_user_id, :from_user_name, :to_user_id, :to_user_name,
  422. :reason, :tenant_no, :order_num, :remark, :create_by, :create_by_name, :create_dept_id,
  423. :create_time, :update_by, :update_time, :del_flag
  424. )
  425. """
  426. )
  427. STEP_INSERT_SQL = {
  428. "crm_customer": INSERT_CUSTOMER_SQL,
  429. "crm_contact": INSERT_CONTACT_SQL,
  430. "crm_customer_follow": INSERT_FOLLOW_SQL,
  431. "crm_customer_pool": INSERT_POOL_SQL,
  432. }
  433. def write_mapping_csv(table_key: str, rows: List[dict], path: Path) -> None:
  434. path.parent.mkdir(parents=True, exist_ok=True)
  435. migrated_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
  436. with path.open("w", encoding="utf-8", newline="") as f:
  437. w = csv.DictWriter(f, fieldnames=["source_id", "new_id", "migrated_at"])
  438. w.writeheader()
  439. for row in rows:
  440. sid = legacy_id_to_str(row["id"])
  441. w.writerow({"source_id": sid, "new_id": sid, "migrated_at": migrated_at})
  442. def migrate_table(
  443. source_engine: Engine,
  444. target_engine: Engine,
  445. table: str,
  446. *,
  447. tenant_no: str,
  448. crm_user_map: Dict[int, str],
  449. admin_id: Optional[str],
  450. oss_map: Dict[str, str],
  451. stats: MigrationStats,
  452. dry_run: bool,
  453. update_existing: bool,
  454. include_deleted: bool,
  455. ) -> List[dict]:
  456. step = stats.step(table)
  457. source_rows = fetch_source_rows(source_engine, table, include_deleted)
  458. step.source_count = len(source_rows)
  459. transformers = {
  460. "crm_customer": lambda r: transform_customer(r, tenant_no, crm_user_map, admin_id, oss_map, stats),
  461. "crm_contact": lambda r: transform_contact(r, tenant_no, crm_user_map, admin_id, stats),
  462. "crm_customer_follow": lambda r: transform_follow(r, tenant_no, crm_user_map, admin_id, oss_map, stats),
  463. "crm_customer_pool": lambda r: transform_pool(r, tenant_no, crm_user_map, admin_id, stats),
  464. }
  465. payloads = [transformers[table](r) for r in source_rows]
  466. if dry_run:
  467. step.inserted = len(payloads)
  468. return source_rows
  469. insert_sql = STEP_INSERT_SQL[table]
  470. with target_engine.begin() as conn:
  471. for payload in payloads:
  472. rid = payload["id"]
  473. if target_row_exists(conn, table, rid):
  474. if update_existing:
  475. # 简化:仅支持首次灌库;更新逻辑后续按需补充
  476. step.skipped += 1
  477. else:
  478. step.skipped += 1
  479. else:
  480. conn.execute(insert_sql, payload)
  481. step.inserted += 1
  482. return source_rows
  483. def print_report(stats: MigrationStats, dry_run: bool) -> None:
  484. mode = "DRY-RUN" if dry_run else "COMMITTED"
  485. print(f"\n=== CRM 客户四表迁移 [{mode}] ===")
  486. for table, _ in MIGRATION_STEPS:
  487. s = stats.step(table)
  488. print(f"{table}: 源={s.source_count} 插入={s.inserted} 跳过={s.skipped}")
  489. if stats.warnings:
  490. print(f"警告 ({len(stats.warnings)}):")
  491. for w in stats.warnings[:20]:
  492. print(f" - {w}")
  493. if len(stats.warnings) > 20:
  494. print(f" ... 另有 {len(stats.warnings) - 20} 条")
  495. def run_migration(
  496. source_engine: Engine,
  497. target_engine: Engine,
  498. *,
  499. tenant_no: str,
  500. user_mapping_path: Path,
  501. mapping_dir: Path,
  502. dry_run: bool,
  503. update_existing: bool,
  504. include_deleted: bool,
  505. only_tables: Optional[List[str]],
  506. ) -> MigrationStats:
  507. stats = MigrationStats()
  508. crm_user_map, admin_id = load_user_mapping(user_mapping_path)
  509. if not admin_id:
  510. admin_id = fetch_portal_admin_id(target_engine)
  511. oss_map = load_oss_id_to_url(target_engine, tenant_no)
  512. print(f"sys_oss 映射条数: {len(oss_map)}")
  513. selected = [t for t, _ in MIGRATION_STEPS if not only_tables or t in only_tables]
  514. for table in selected:
  515. print(f"\n>> 迁移 {table} ...")
  516. source_rows = migrate_table(
  517. source_engine,
  518. target_engine,
  519. table,
  520. tenant_no=tenant_no,
  521. crm_user_map=crm_user_map,
  522. admin_id=admin_id,
  523. oss_map=oss_map,
  524. stats=stats,
  525. dry_run=dry_run,
  526. update_existing=update_existing,
  527. include_deleted=include_deleted,
  528. )
  529. if table == "crm_customer" and not dry_run:
  530. write_mapping_csv("customer", source_rows, mapping_dir / "crm_customer_id_mapping.csv")
  531. print_report(stats, dry_run)
  532. return stats
  533. def main() -> int:
  534. parser = argparse.ArgumentParser(description="Migrate CRM customer bundle from sx to portal")
  535. parser.add_argument("--source-url", default="")
  536. parser.add_argument("--target-url", default="")
  537. parser.add_argument("--tenant-no", default=DEFAULT_TENANT_NO)
  538. parser.add_argument(
  539. "--user-mapping",
  540. default=str(BACKEND_ROOT.parent / "docs/data-migration/mappings/user_mapping_result.csv"),
  541. )
  542. parser.add_argument("--dry-run", action="store_true")
  543. parser.add_argument("--update", action="store_true")
  544. parser.add_argument("--include-deleted", action="store_true")
  545. parser.add_argument("--tables", default="", help="逗号分隔,默认全部四表")
  546. parser.add_argument("--env-file", default=os.environ.get("ENV_FILE", ""))
  547. args = parser.parse_args()
  548. load_settings_env(args.env_file or None)
  549. settings = load_runtime_settings()
  550. target_url = args.target_url or settings.DATABASE_URL
  551. source_url = args.source_url or os.environ.get("SX_SOURCE_DATABASE_URL") or derive_source_url(target_url)
  552. only_tables = [t.strip() for t in args.tables.split(",") if t.strip()] or None
  553. print(f"源库: {urlparse(source_url).path}")
  554. print(f"目标库: {urlparse(target_url).path}")
  555. run_migration(
  556. create_engine(source_url, pool_pre_ping=True),
  557. create_engine(target_url, pool_pre_ping=True),
  558. tenant_no=args.tenant_no,
  559. user_mapping_path=Path(args.user_mapping),
  560. mapping_dir=BACKEND_ROOT.parent / "docs/data-migration/mappings",
  561. dry_run=args.dry_run,
  562. update_existing=args.update,
  563. include_deleted=args.include_deleted,
  564. only_tables=only_tables,
  565. )
  566. return 0
  567. if __name__ == "__main__":
  568. raise SystemExit(main())