test_migration_clear_modes_disabled.py 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184
  1. """Offline restoration regressions; extract functions, never import scripts or run main.
  2. The historical filename remains so the session's blanket-disable tests are replaced
  3. in place. Explicit migration clear modes retain their original dry-run behavior.
  4. """
  5. import argparse
  6. import ast
  7. from collections import defaultdict
  8. import os
  9. from pathlib import Path
  10. import sys
  11. from types import SimpleNamespace
  12. import pytest
  13. SCRIPTS = Path(__file__).resolve().parents[1] / 'scripts'
  14. CASES = [
  15. ('migrate_crm_product.py', 'run_migration', 'clear_target', 'clear_target_products'),
  16. ('migrate_crm_product_category.py', 'run_migration', 'clear_target', 'clear_target_categories'),
  17. ('migrate_dibang_data.py', 'migrate', 'truncate_target', 'truncate_target_tables'),
  18. ('migrate_bx_data.py', 'migrate_table', 'replace_workflow', 'clear_portal_workflow'),
  19. ]
  20. def tree(filename):
  21. return ast.parse((SCRIPTS / filename).read_text())
  22. def function_node(filename, name):
  23. return next(n for n in tree(filename).body if isinstance(n, ast.FunctionDef) and n.name == name)
  24. def extract(filename, name, namespace):
  25. module = ast.Module(body=[ast.ImportFrom(module='__future__', names=[ast.alias(name='annotations')], level=0), function_node(filename, name)], type_ignores=[])
  26. exec(compile(ast.fix_missing_locations(module), filename, 'exec'), namespace)
  27. return namespace[name]
  28. class FakeEngine:
  29. """Records transaction/SQL intent without creating any database engine."""
  30. def __init__(self):
  31. self.statements = []
  32. self.transactions = 0
  33. def begin(self):
  34. self.transactions += 1
  35. return self
  36. def __enter__(self):
  37. return self
  38. def __exit__(self, *args):
  39. return False
  40. def execute(self, sql, params=None):
  41. self.statements.append((sql, params))
  42. return SimpleNamespace(rowcount=2)
  43. class Stats:
  44. def __init__(self):
  45. self.source_counts = {}
  46. self.inserted = {}
  47. self.steps = defaultdict(lambda: SimpleNamespace(source_count=0, inserted=0, skipped=0))
  48. def step(self, table):
  49. return self.steps[table]
  50. def namespace_for(filename, helper):
  51. keys = ('purchase_plans', 'inbound_outbound', 'weighing_slips')
  52. namespace = dict(
  53. text=lambda sql: sql, TARGET_TABLE='crm_test',
  54. TARGET_TABLES={key: 'db_' + key for key in keys},
  55. SOURCE_TABLES={key: key for key in keys},
  56. TABLE_OPTIONS={}, MigrationStats=Stats,
  57. load_user_mapping=lambda _: ({}, 'admin'),
  58. load_bx_user_mapping=lambda _: ({}, 'admin'),
  59. fetch_source_rows=lambda *a, **kw: [], fetch_rows=lambda *a: [],
  60. topo_sort_categories=lambda rows: rows,
  61. build_id_maps=lambda rows: {key: {} for key in rows},
  62. print_report=lambda *a, **kw: None,
  63. )
  64. extract(filename, helper, namespace)
  65. return namespace
  66. @pytest.mark.parametrize('filename,name,flag,helper', CASES)
  67. @pytest.mark.parametrize('dry_run', [False, True])
  68. @pytest.mark.parametrize('clear', [False, True])
  69. def test_original_clear_mode_and_dry_run(filename, name, flag, helper, dry_run, clear):
  70. import inspect
  71. namespace = namespace_for(filename, helper)
  72. fn = extract(filename, name, namespace)
  73. kwargs = {key: None for key in inspect.signature(fn).parameters}
  74. target = FakeEngine()
  75. kwargs.update(source_engine=FakeEngine(), target_engine=target, tenant_no='tenant', dry_run=dry_run)
  76. kwargs[flag] = clear
  77. if name == 'migrate_table':
  78. kwargs.update(table='bx_workflow_def', stats=Stats())
  79. fn(**kwargs)
  80. assert target.transactions == (0 if dry_run else 1)
  81. expected = 0 if dry_run or not clear else (2 if filename == 'migrate_bx_data.py' else 3 if filename == 'migrate_dibang_data.py' else 1)
  82. assert len(target.statements) == expected
  83. @pytest.mark.parametrize('filename,name,flag,helper', CASES)
  84. def test_original_clear_helpers_execute_expected_sql_offline(filename, name, flag, helper):
  85. namespace = namespace_for(filename, helper)
  86. target = FakeEngine()
  87. fn = namespace[helper]
  88. if filename == 'migrate_bx_data.py':
  89. fn(target)
  90. assert target.statements == [('DELETE FROM bx_workflow_node', None), ('DELETE FROM bx_workflow_def', None)]
  91. else:
  92. result = fn(target, 'tenant')
  93. assert all('DELETE FROM' in sql for sql, _ in target.statements)
  94. key = 'tenant_no' if filename == 'migrate_dibang_data.py' else 't'
  95. assert all(params == {key: 'tenant'} for _, params in target.statements)
  96. if key == 't':
  97. assert result == 2
  98. assert "tenant_no='' OR tenant_no IS NULL" in target.statements[0][0]
  99. def cli_parser(filename):
  100. """Execute only argparse setup, stopping immediately after parse_args."""
  101. node = function_node(filename, 'main')
  102. statements = []
  103. for statement in node.body:
  104. statements.append(statement)
  105. if isinstance(statement, ast.Assign) and any(isinstance(t, ast.Name) and t.id == 'args' for t in statement.targets):
  106. break
  107. else:
  108. raise AssertionError('parse_args not found')
  109. node.name = 'parse_only'
  110. node.body = statements + [ast.Return(value=ast.Name(id='args', ctx=ast.Load()))]
  111. namespace = dict(argparse=argparse, os=os, sys=sys, Path=Path, DEFAULT_TENANT_NO='t', BACKEND_ROOT=SCRIPTS.parent)
  112. exec(compile(ast.fix_missing_locations(ast.Module(body=[node], type_ignores=[])), filename, 'exec'), namespace)
  113. return namespace['parse_only']
  114. @pytest.mark.parametrize('filename,flag,attribute', [
  115. ('migrate_crm_product.py', '--clear-target', 'clear_target'),
  116. ('migrate_crm_product_category.py', '--clear-target', 'clear_target'),
  117. ('migrate_dibang_data.py', '--truncate-target', 'truncate_target'),
  118. ])
  119. def test_cli_preserves_explicit_clear_option(filename, flag, attribute, monkeypatch):
  120. monkeypatch.setattr(sys, 'argv', ['offline-test', flag])
  121. assert getattr(cli_parser(filename)(), attribute) is True
  122. main = function_node(filename, 'main')
  123. assert not any(isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) and n.func.attr == 'error' for n in ast.walk(main))
  124. def test_dibang_cli_keeps_dry_run_clear_conflict_guard():
  125. main = function_node('migrate_dibang_data.py', 'main')
  126. guards = [n for n in main.body if isinstance(n, ast.If) and 'args.truncate_target' in ast.unparse(n.test) and 'args.dry_run' in ast.unparse(n.test)]
  127. assert len(guards) == 1
  128. assert any(isinstance(n, ast.Return) and isinstance(n.value, ast.Constant) and n.value.value == 2 for n in ast.walk(guards[0]))
  129. @pytest.mark.parametrize('legacy_flag', [False, True])
  130. def test_bx_cli_preserves_original_replace_default_and_opt_out(legacy_flag, monkeypatch):
  131. monkeypatch.setattr(sys, 'argv', ['offline-test'] + (['--no-replace-workflow'] if legacy_flag else []))
  132. args = cli_parser('migrate_bx_data.py')()
  133. main = function_node('migrate_bx_data.py', 'main')
  134. assignment = next(n for n in main.body if isinstance(n, ast.Assign) and any(isinstance(t, ast.Name) and t.id == 'replace_workflow' for t in n.targets))
  135. assert eval(compile(ast.Expression(assignment.value), '<offline-default>', 'eval'), {'args': args}) is (not legacy_flag)
  136. @pytest.mark.parametrize('dry_run', [False, True])
  137. @pytest.mark.parametrize('replace', [False, True])
  138. def test_bx_run_migration_forwards_replace_and_dry_run(dry_run, replace):
  139. calls = []
  140. namespace = dict(MigrationStats=Stats, load_bx_user_mapping=lambda _: ({}, 'admin'),
  141. resolve_steps=lambda *a: ['bx_workflow_def'],
  142. migrate_table=lambda *a, **kw: calls.append(kw) or [],
  143. print_report=lambda *a: None)
  144. fn = extract('migrate_bx_data.py', 'run_migration', namespace)
  145. fn(FakeEngine(), FakeEngine(), tenant_no='tenant', user_mapping_path=None,
  146. mapping_dir=None, dry_run=dry_run, phase='base', only_tables=None,
  147. replace_workflow=replace)
  148. assert len(calls) == 1
  149. assert calls[0]['replace_workflow'] is replace
  150. assert calls[0]['dry_run'] is dry_run