| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655 |
- #!/usr/bin/env python3
- """将 sx 库 CRM 客户相关表迁入 shanxia2.0。
- 顺序:crm_customer → crm_contact → crm_customer_follow → crm_customer_pool
- 证照字段 OSS ID → 目标 sys_oss 相对 URL;用户外键经 user_mapping 转换。
- 用法:
- cd backend
- venv/bin/python scripts/migrate_crm_customer.py --dry-run --env-file .env.production
- venv/bin/python scripts/migrate_crm_customer.py --env-file .env.production
- """
- from __future__ import annotations
- import argparse
- import csv
- import os
- import sys
- from dataclasses import dataclass, field
- from datetime import datetime
- from decimal import Decimal
- from pathlib import Path
- from typing import Any, Dict, List, Optional, Tuple
- from urllib.parse import urlparse, urlunparse
- from sqlalchemy import create_engine, text
- from sqlalchemy.engine import Engine
- BACKEND_ROOT = Path(__file__).resolve().parents[1]
- MIGRATION_DIR = BACKEND_ROOT / "scripts" / "migration"
- if str(BACKEND_ROOT) not in sys.path:
- sys.path.insert(0, str(BACKEND_ROOT))
- if str(MIGRATION_DIR) not in sys.path:
- sys.path.insert(0, str(MIGRATION_DIR))
- from minio_url import resolve_oss_id_csv # noqa: E402
- DEFAULT_TENANT_NO = "shanxia2.0"
- CRM_ADMIN_SOURCE_USER_ID = 1
- CUSTOMER_TYPE_MAP = {
- "company": "company",
- "person": "person",
- "未分类": None,
- "a类国央企单位": "A类国央企业",
- "A类国央企单位": "A类国央企业",
- "A类国央企业": "A类国央企业",
- "b类股份制企业": "B类股份制企业",
- "B类股份制企业": "B类股份制企业",
- "c类私营单位": "C类私营企业",
- "C类私营企业": "C类私营企业",
- "d外商投资企业": "D外商投资企业",
- "D外商投资企业": "D外商投资企业",
- "e类国资委企业": "5",
- "5": "5",
- }
- FOLLOW_TYPE_MAP = {
- "电话": "phone",
- "phone": "phone",
- "微信": "wechat",
- "wechat": "wechat",
- "邮件": "email",
- "email": "email",
- "拜访": "visit",
- "visit": "visit",
- "其他": "other",
- "other": "other",
- }
- MIGRATION_STEPS = (
- ("crm_customer", "customer"),
- ("crm_contact", "contact"),
- ("crm_customer_follow", "follow"),
- ("crm_customer_pool", "pool"),
- )
- @dataclass
- class StepStats:
- source_count: int = 0
- inserted: int = 0
- updated: int = 0
- skipped: int = 0
- @dataclass
- class MigrationStats:
- steps: Dict[str, StepStats] = field(default_factory=dict)
- warnings: List[str] = field(default_factory=list)
- def step(self, name: str) -> StepStats:
- if name not in self.steps:
- self.steps[name] = StepStats()
- return self.steps[name]
- def legacy_id_to_str(legacy_id: Any) -> Optional[str]:
- if legacy_id is None or str(legacy_id).strip() == "":
- return None
- try:
- return str(int(legacy_id))
- except (TypeError, ValueError):
- return str(legacy_id).strip()
- def derive_source_url(target_url: str, source_db: str = "sx") -> str:
- return urlunparse(urlparse(target_url)._replace(path=f"/{source_db}"))
- def load_settings_env(env_file: Optional[str]) -> None:
- if env_file:
- os.environ["ENV_FILE"] = env_file
- from dotenv import load_dotenv
- candidates: List[Path] = []
- if env_file:
- p = Path(env_file)
- candidates.append(p if p.is_absolute() else BACKEND_ROOT / p)
- candidates.extend([BACKEND_ROOT / ".env.production", BACKEND_ROOT / ".env"])
- for path in candidates:
- if path.is_file():
- load_dotenv(path, override=True)
- break
- def load_runtime_settings():
- from app.core.config import Settings
- return Settings()
- def load_user_mapping(path: Path) -> Tuple[Dict[int, str], Optional[str]]:
- crm_map: Dict[int, str] = {}
- admin_id: Optional[str] = None
- with path.open(encoding="utf-8-sig", newline="") as f:
- for row in csv.DictReader(f):
- new_id = (row.get("new_user_id") or "").strip()
- user_no = (row.get("user_no") or "").strip()
- crm_raw = (row.get("crm_user_id") or "").strip()
- if user_no == "admin" and new_id:
- admin_id = new_id
- if crm_raw and new_id:
- try:
- crm_map[int(crm_raw)] = new_id
- except ValueError:
- pass
- return crm_map, admin_id
- def fetch_portal_admin_id(target_engine: Engine) -> Optional[str]:
- with target_engine.connect() as conn:
- row = conn.execute(
- text("SELECT id FROM sys_user WHERE user_no='admin' AND del_flag=0 LIMIT 1")
- ).first()
- return str(row[0]) if row and row[0] else None
- def resolve_portal_user_id(
- source_user_id: Any,
- crm_user_map: Dict[int, str],
- admin_id: Optional[str],
- stats: MigrationStats,
- label: str,
- *,
- required: bool = False,
- ) -> Optional[str]:
- if source_user_id is None or str(source_user_id).strip() == "":
- return None
- try:
- uid = int(source_user_id)
- except (TypeError, ValueError):
- stats.warnings.append(f"{label}: 非法用户 id {source_user_id!r}")
- return None
- if uid == CRM_ADMIN_SOURCE_USER_ID:
- return admin_id
- mapped = crm_user_map.get(uid)
- if not mapped:
- msg = f"{label}: crm_user_id={uid} 无映射"
- if required:
- stats.warnings.append(msg)
- return mapped
- def load_oss_id_to_url(target_engine: Engine, tenant_no: str) -> Dict[str, str]:
- with target_engine.connect() as conn:
- rows = conn.execute(
- text("SELECT id, url FROM sys_oss WHERE tenant_no=:t AND del_flag=0"),
- {"t": tenant_no},
- ).mappings().all()
- return {str(r["id"]): (r["url"] or "").strip() for r in rows if r["id"]}
- def normalize_customer_type(raw: Any, stats: MigrationStats, label: str) -> Optional[str]:
- value = (str(raw).strip() if raw is not None else "") or ""
- if not value:
- return None
- if value in CUSTOMER_TYPE_MAP:
- mapped = CUSTOMER_TYPE_MAP[value]
- if mapped is None:
- stats.warnings.append(f"{label}: customer_type={value!r} 已置空")
- return mapped
- lowered = value.lower()
- for key, mapped in CUSTOMER_TYPE_MAP.items():
- if key.lower() == lowered:
- return mapped
- stats.warnings.append(f"{label}: customer_type={value!r} 未识别,已置空")
- return None
- def normalize_follow_type(raw: Any) -> str:
- value = (str(raw).strip() if raw is not None else "") or "other"
- return FOLLOW_TYPE_MAP.get(value, FOLLOW_TYPE_MAP.get(value.lower(), "other"))
- def _dec(value: Any) -> Optional[Decimal]:
- if value is None or value == "":
- return None
- return Decimal(str(value))
- def _str_or_none(value: Any, max_len: int = 0) -> Optional[str]:
- if value is None:
- return None
- s = str(value).strip()
- if not s:
- return None
- return s[:max_len] if max_len else s
- def fetch_source_rows(engine: Engine, table: str, include_deleted: bool) -> List[dict]:
- where = "" if include_deleted else "WHERE del_flag = 0"
- sql = f"SELECT * FROM `{table}` {where} ORDER BY id ASC"
- with engine.connect() as conn:
- return [dict(r._mapping) for r in conn.execute(text(sql))]
- def target_row_exists(conn, table: str, row_id: str) -> bool:
- return int(conn.execute(text(f"SELECT COUNT(*) FROM `{table}` WHERE id=:id"), {"id": row_id}).scalar_one()) > 0
- def transform_customer(
- row: dict,
- tenant_no: str,
- crm_user_map: Dict[int, str],
- admin_id: Optional[str],
- oss_map: Dict[str, str],
- stats: MigrationStats,
- ) -> dict:
- rid = legacy_id_to_str(row["id"])
- label = f"customer#{rid}"
- def map_user(field_val: Any) -> Optional[str]:
- return resolve_portal_user_id(field_val, crm_user_map, admin_id, stats, f"{label} {field_val}")
- return {
- "id": rid,
- "customer_no": (row.get("customer_no") or "")[:50],
- "customer_name": (row.get("customer_name") or "")[:200],
- "short_name": _str_or_none(row.get("short_name"), 100),
- "customer_type": normalize_customer_type(row.get("customer_type"), stats, label),
- "mobile": _str_or_none(row.get("mobile"), 20),
- "telephone": _str_or_none(row.get("telephone"), 30),
- "email": _str_or_none(row.get("email"), 100),
- "website": _str_or_none(row.get("website"), 500),
- "level": (row.get("level") or "normal")[:20],
- "source": _str_or_none(row.get("source"), 20),
- "industry": _str_or_none(row.get("industry"), 50),
- "park_area": _str_or_none(row.get("park_area"), 100),
- "payment_terms": _str_or_none(row.get("payment_terms"), 500),
- "scale": _str_or_none(row.get("scale"), 20),
- "province": _str_or_none(row.get("province"), 50),
- "city": _str_or_none(row.get("city"), 50),
- "district": _str_or_none(row.get("district"), 50),
- "address": _str_or_none(row.get("address"), 255),
- "status": (row.get("status") or "normal")[:20],
- "deal_status": (row.get("deal_status") or "not_deal")[:20],
- "owner_user_id": map_user(row.get("owner_user_id")),
- "owner_user_name": _str_or_none(row.get("owner_user_name"), 50),
- "last_time": row.get("last_time"),
- "next_time": row.get("next_time"),
- "follow_count": int(row.get("follow_count") or 0),
- "deal_time": row.get("deal_time"),
- "deal_amount": _dec(row.get("deal_amount")) or Decimal("0.00"),
- "contact_count": int(row.get("contact_count") or 0),
- "opportunity_count": int(row.get("opportunity_count") or 0),
- "contract_count": int(row.get("contract_count") or 0),
- "into_pool_time": row.get("into_pool_time"),
- "into_pool_by": map_user(row.get("into_pool_by")),
- "into_pool_reason": _str_or_none(row.get("into_pool_reason"), 500),
- "claim_time": row.get("claim_time"),
- "assign_time": row.get("assign_time"),
- "assign_by": map_user(row.get("assign_by")),
- "protect_time": row.get("protect_time"),
- "business_license_img": resolve_oss_id_csv(row.get("business_license_images"), oss_map) or None,
- "business_permit_images": resolve_oss_id_csv(row.get("business_permit_images"), oss_map) or None,
- "other_cert_images": resolve_oss_id_csv(row.get("other_cert_images"), oss_map) or None,
- "clue_id": legacy_id_to_str(row.get("clue_id")),
- "tenant_no": tenant_no,
- "order_num": 0,
- "remark": _str_or_none(row.get("remark"), 512),
- "create_by": map_user(row.get("create_by")),
- "create_dept_id": "",
- "create_time": row.get("create_time") or datetime.now(),
- "update_by": map_user(row.get("update_by")),
- "update_time": row.get("update_time") or row.get("create_time") or datetime.now(),
- "del_flag": int(row.get("del_flag") or 0),
- }
- def transform_contact(
- row: dict,
- tenant_no: str,
- crm_user_map: Dict[int, str],
- admin_id: Optional[str],
- stats: MigrationStats,
- ) -> dict:
- rid = legacy_id_to_str(row["id"])
- label = f"contact#{rid}"
- def map_user(field_val: Any) -> Optional[str]:
- return resolve_portal_user_id(field_val, crm_user_map, admin_id, stats, label)
- return {
- "id": rid,
- "customer_id": legacy_id_to_str(row.get("customer_id")),
- "customer_name": _str_or_none(row.get("customer_name"), 200),
- "contact_name": (row.get("contact_name") or "")[:100],
- "gender": _str_or_none(row.get("gender"), 10),
- "mobile": _str_or_none(row.get("mobile"), 20),
- "telephone": _str_or_none(row.get("telephone"), 30),
- "email": _str_or_none(row.get("email"), 100),
- "wechat": _str_or_none(row.get("wechat"), 100),
- "qq": _str_or_none(row.get("qq"), 20),
- "department": _str_or_none(row.get("department"), 100),
- "position": _str_or_none(row.get("position"), 50),
- "role": _str_or_none(row.get("role"), 20),
- "birthday": row.get("birthday"),
- "address": _str_or_none(row.get("address"), 255),
- "is_key": int(row.get("is_key") or 0),
- "is_primary": int(row.get("is_primary") or 0),
- "status": (row.get("status") or "active")[:20],
- "tenant_no": tenant_no,
- "order_num": 0,
- "remark": _str_or_none(row.get("remark"), 512),
- "create_by": map_user(row.get("create_by")),
- "create_dept_id": "",
- "create_time": row.get("create_time") or datetime.now(),
- "update_by": map_user(row.get("update_by")),
- "update_time": row.get("update_time") or row.get("create_time") or datetime.now(),
- "del_flag": int(row.get("del_flag") or 0),
- }
- def transform_follow(
- row: dict,
- tenant_no: str,
- crm_user_map: Dict[int, str],
- admin_id: Optional[str],
- oss_map: Dict[str, str],
- stats: MigrationStats,
- ) -> dict:
- rid = legacy_id_to_str(row["id"])
- label = f"follow#{rid}"
- owner_id = resolve_portal_user_id(row.get("create_by"), crm_user_map, admin_id, stats, label)
- attachment = resolve_oss_id_csv(row.get("attachments"), oss_map) or None
- return {
- "id": rid,
- "customer_id": legacy_id_to_str(row.get("customer_id")),
- "customer_name": _str_or_none(row.get("customer_name"), 200),
- "contact_id": legacy_id_to_str(row.get("contact_id")),
- "contact_name": _str_or_none(row.get("contact_name"), 100),
- "follow_type": normalize_follow_type(row.get("follow_type")),
- "follow_content": row.get("follow_content") or "",
- "follow_time": row.get("create_time"),
- "next_time": row.get("next_time"),
- "attachment": attachment,
- "owner_user_id": owner_id,
- "owner_user_name": _str_or_none(row.get("create_by_name"), 50),
- "tenant_no": tenant_no,
- "order_num": 0,
- "remark": _str_or_none(row.get("remark"), 512),
- "create_by": owner_id,
- "create_dept_id": "",
- "create_time": row.get("create_time") or datetime.now(),
- "update_by": resolve_portal_user_id(row.get("update_by"), crm_user_map, admin_id, stats, label),
- "update_time": row.get("update_time") or row.get("create_time") or datetime.now(),
- "del_flag": int(row.get("del_flag") or 0),
- }
- def transform_pool(
- row: dict,
- tenant_no: str,
- crm_user_map: Dict[int, str],
- admin_id: Optional[str],
- stats: MigrationStats,
- ) -> dict:
- rid = legacy_id_to_str(row["id"])
- label = f"pool#{rid}"
- def map_user(field_val: Any) -> Optional[str]:
- return resolve_portal_user_id(field_val, crm_user_map, admin_id, stats, label)
- create_by = map_user(row.get("create_by"))
- return {
- "id": rid,
- "customer_id": legacy_id_to_str(row.get("customer_id")),
- "customer_name": _str_or_none(row.get("customer_name"), 200),
- "operation_type": (row.get("operation_type") or "")[:20],
- "from_user_id": map_user(row.get("from_user_id")),
- "from_user_name": _str_or_none(row.get("from_user_name"), 50),
- "to_user_id": map_user(row.get("to_user_id")),
- "to_user_name": _str_or_none(row.get("to_user_name"), 50),
- "reason": _str_or_none(row.get("reason"), 500),
- "tenant_no": tenant_no,
- "order_num": 0,
- "remark": _str_or_none(row.get("remark"), 500),
- "create_by": create_by,
- "create_by_name": _str_or_none(row.get("create_by_name"), 50),
- "create_dept_id": "",
- "create_time": row.get("create_time") or datetime.now(),
- "update_by": map_user(row.get("update_by")),
- "update_time": row.get("update_time") or row.get("create_time") or datetime.now(),
- "del_flag": int(row.get("del_flag") or 0),
- }
- INSERT_CUSTOMER_SQL = text(
- """
- INSERT INTO crm_customer (
- id, customer_no, customer_name, short_name, customer_type, mobile, telephone, email, website,
- level, source, industry, park_area, payment_terms, scale, province, city, district, address,
- status, deal_status, owner_user_id, owner_user_name, last_time, next_time, follow_count,
- deal_time, deal_amount, contact_count, opportunity_count, contract_count, into_pool_time,
- into_pool_by, into_pool_reason, claim_time, assign_time, assign_by, protect_time,
- business_license_img, business_permit_images, other_cert_images, clue_id,
- tenant_no, order_num, remark, create_by, create_dept_id, create_time, update_by, update_time, del_flag
- ) VALUES (
- :id, :customer_no, :customer_name, :short_name, :customer_type, :mobile, :telephone, :email, :website,
- :level, :source, :industry, :park_area, :payment_terms, :scale, :province, :city, :district, :address,
- :status, :deal_status, :owner_user_id, :owner_user_name, :last_time, :next_time, :follow_count,
- :deal_time, :deal_amount, :contact_count, :opportunity_count, :contract_count, :into_pool_time,
- :into_pool_by, :into_pool_reason, :claim_time, :assign_time, :assign_by, :protect_time,
- :business_license_img, :business_permit_images, :other_cert_images, :clue_id,
- :tenant_no, :order_num, :remark, :create_by, :create_dept_id, :create_time, :update_by, :update_time, :del_flag
- )
- """
- )
- INSERT_CONTACT_SQL = text(
- """
- INSERT INTO crm_contact (
- id, customer_id, customer_name, contact_name, gender, mobile, telephone, email, wechat, qq,
- department, position, role, birthday, address, is_key, is_primary, status,
- tenant_no, order_num, remark, create_by, create_dept_id, create_time, update_by, update_time, del_flag
- ) VALUES (
- :id, :customer_id, :customer_name, :contact_name, :gender, :mobile, :telephone, :email, :wechat, :qq,
- :department, :position, :role, :birthday, :address, :is_key, :is_primary, :status,
- :tenant_no, :order_num, :remark, :create_by, :create_dept_id, :create_time, :update_by, :update_time, :del_flag
- )
- """
- )
- INSERT_FOLLOW_SQL = text(
- """
- INSERT INTO crm_customer_follow (
- id, customer_id, customer_name, contact_id, contact_name, follow_type, follow_content,
- follow_time, next_time, attachment, owner_user_id, owner_user_name,
- tenant_no, order_num, remark, create_by, create_dept_id, create_time, update_by, update_time, del_flag
- ) VALUES (
- :id, :customer_id, :customer_name, :contact_id, :contact_name, :follow_type, :follow_content,
- :follow_time, :next_time, :attachment, :owner_user_id, :owner_user_name,
- :tenant_no, :order_num, :remark, :create_by, :create_dept_id, :create_time, :update_by, :update_time, :del_flag
- )
- """
- )
- INSERT_POOL_SQL = text(
- """
- INSERT INTO crm_customer_pool (
- id, customer_id, customer_name, operation_type, from_user_id, from_user_name, to_user_id, to_user_name,
- reason, tenant_no, order_num, remark, create_by, create_by_name, create_dept_id,
- create_time, update_by, update_time, del_flag
- ) VALUES (
- :id, :customer_id, :customer_name, :operation_type, :from_user_id, :from_user_name, :to_user_id, :to_user_name,
- :reason, :tenant_no, :order_num, :remark, :create_by, :create_by_name, :create_dept_id,
- :create_time, :update_by, :update_time, :del_flag
- )
- """
- )
- STEP_INSERT_SQL = {
- "crm_customer": INSERT_CUSTOMER_SQL,
- "crm_contact": INSERT_CONTACT_SQL,
- "crm_customer_follow": INSERT_FOLLOW_SQL,
- "crm_customer_pool": INSERT_POOL_SQL,
- }
- def write_mapping_csv(table_key: str, rows: List[dict], path: Path) -> None:
- path.parent.mkdir(parents=True, exist_ok=True)
- migrated_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
- with path.open("w", encoding="utf-8", newline="") as f:
- w = csv.DictWriter(f, fieldnames=["source_id", "new_id", "migrated_at"])
- w.writeheader()
- for row in rows:
- sid = legacy_id_to_str(row["id"])
- w.writerow({"source_id": sid, "new_id": sid, "migrated_at": migrated_at})
- def migrate_table(
- source_engine: Engine,
- target_engine: Engine,
- table: str,
- *,
- tenant_no: str,
- crm_user_map: Dict[int, str],
- admin_id: Optional[str],
- oss_map: Dict[str, str],
- stats: MigrationStats,
- dry_run: bool,
- update_existing: bool,
- include_deleted: bool,
- ) -> List[dict]:
- step = stats.step(table)
- source_rows = fetch_source_rows(source_engine, table, include_deleted)
- step.source_count = len(source_rows)
- transformers = {
- "crm_customer": lambda r: transform_customer(r, tenant_no, crm_user_map, admin_id, oss_map, stats),
- "crm_contact": lambda r: transform_contact(r, tenant_no, crm_user_map, admin_id, stats),
- "crm_customer_follow": lambda r: transform_follow(r, tenant_no, crm_user_map, admin_id, oss_map, stats),
- "crm_customer_pool": lambda r: transform_pool(r, tenant_no, crm_user_map, admin_id, stats),
- }
- payloads = [transformers[table](r) for r in source_rows]
- if dry_run:
- step.inserted = len(payloads)
- return source_rows
- insert_sql = STEP_INSERT_SQL[table]
- with target_engine.begin() as conn:
- for payload in payloads:
- rid = payload["id"]
- if target_row_exists(conn, table, rid):
- if update_existing:
- # 简化:仅支持首次灌库;更新逻辑后续按需补充
- step.skipped += 1
- else:
- step.skipped += 1
- else:
- conn.execute(insert_sql, payload)
- step.inserted += 1
- return source_rows
- def print_report(stats: MigrationStats, dry_run: bool) -> None:
- mode = "DRY-RUN" if dry_run else "COMMITTED"
- print(f"\n=== CRM 客户四表迁移 [{mode}] ===")
- for table, _ in MIGRATION_STEPS:
- s = stats.step(table)
- print(f"{table}: 源={s.source_count} 插入={s.inserted} 跳过={s.skipped}")
- if stats.warnings:
- print(f"警告 ({len(stats.warnings)}):")
- for w in stats.warnings[:20]:
- print(f" - {w}")
- if len(stats.warnings) > 20:
- print(f" ... 另有 {len(stats.warnings) - 20} 条")
- def run_migration(
- source_engine: Engine,
- target_engine: Engine,
- *,
- tenant_no: str,
- user_mapping_path: Path,
- mapping_dir: Path,
- dry_run: bool,
- update_existing: bool,
- include_deleted: bool,
- only_tables: Optional[List[str]],
- ) -> MigrationStats:
- stats = MigrationStats()
- crm_user_map, admin_id = load_user_mapping(user_mapping_path)
- if not admin_id:
- admin_id = fetch_portal_admin_id(target_engine)
- oss_map = load_oss_id_to_url(target_engine, tenant_no)
- print(f"sys_oss 映射条数: {len(oss_map)}")
- selected = [t for t, _ in MIGRATION_STEPS if not only_tables or t in only_tables]
- for table in selected:
- print(f"\n>> 迁移 {table} ...")
- source_rows = migrate_table(
- source_engine,
- target_engine,
- table,
- tenant_no=tenant_no,
- crm_user_map=crm_user_map,
- admin_id=admin_id,
- oss_map=oss_map,
- stats=stats,
- dry_run=dry_run,
- update_existing=update_existing,
- include_deleted=include_deleted,
- )
- if table == "crm_customer" and not dry_run:
- write_mapping_csv("customer", source_rows, mapping_dir / "crm_customer_id_mapping.csv")
- print_report(stats, dry_run)
- return stats
- def main() -> int:
- parser = argparse.ArgumentParser(description="Migrate CRM customer bundle from sx to portal")
- parser.add_argument("--source-url", default="")
- parser.add_argument("--target-url", default="")
- parser.add_argument("--tenant-no", default=DEFAULT_TENANT_NO)
- parser.add_argument(
- "--user-mapping",
- default=str(BACKEND_ROOT.parent / "docs/data-migration/mappings/user_mapping_result.csv"),
- )
- parser.add_argument("--dry-run", action="store_true")
- parser.add_argument("--update", action="store_true")
- parser.add_argument("--include-deleted", action="store_true")
- parser.add_argument("--tables", default="", help="逗号分隔,默认全部四表")
- parser.add_argument("--env-file", default=os.environ.get("ENV_FILE", ""))
- args = parser.parse_args()
- load_settings_env(args.env_file or None)
- settings = load_runtime_settings()
- target_url = args.target_url or settings.DATABASE_URL
- source_url = args.source_url or os.environ.get("SX_SOURCE_DATABASE_URL") or derive_source_url(target_url)
- only_tables = [t.strip() for t in args.tables.split(",") if t.strip()] or None
- print(f"源库: {urlparse(source_url).path}")
- print(f"目标库: {urlparse(target_url).path}")
- run_migration(
- create_engine(source_url, pool_pre_ping=True),
- create_engine(target_url, pool_pre_ping=True),
- tenant_no=args.tenant_no,
- user_mapping_path=Path(args.user_mapping),
- mapping_dir=BACKEND_ROOT.parent / "docs/data-migration/mappings",
- dry_run=args.dry_run,
- update_existing=args.update,
- include_deleted=args.include_deleted,
- only_tables=only_tables,
- )
- return 0
- if __name__ == "__main__":
- raise SystemExit(main())
|