sqla_compat.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665
  1. # mypy: allow-untyped-defs, allow-incomplete-defs, allow-untyped-calls
  2. # mypy: no-warn-return-any, allow-any-generics
  3. from __future__ import annotations
  4. import contextlib
  5. import re
  6. from typing import Any
  7. from typing import Callable
  8. from typing import Dict
  9. from typing import Iterable
  10. from typing import Iterator
  11. from typing import Mapping
  12. from typing import Optional
  13. from typing import Protocol
  14. from typing import Set
  15. from typing import Type
  16. from typing import TYPE_CHECKING
  17. from typing import TypeVar
  18. from typing import Union
  19. from sqlalchemy import __version__
  20. from sqlalchemy import inspect
  21. from sqlalchemy import schema
  22. from sqlalchemy import sql
  23. from sqlalchemy import types as sqltypes
  24. from sqlalchemy.engine import url
  25. from sqlalchemy.schema import CheckConstraint
  26. from sqlalchemy.schema import Column
  27. from sqlalchemy.schema import ForeignKeyConstraint
  28. from sqlalchemy.sql import visitors
  29. from sqlalchemy.sql.base import DialectKWArgs
  30. from sqlalchemy.sql.elements import BindParameter
  31. from sqlalchemy.sql.elements import ColumnClause
  32. from sqlalchemy.sql.elements import quoted_name
  33. from sqlalchemy.sql.elements import TextClause
  34. from sqlalchemy.sql.elements import UnaryExpression
  35. from sqlalchemy.sql.visitors import traverse
  36. from typing_extensions import TypeGuard
  37. if TYPE_CHECKING:
  38. from sqlalchemy import ClauseElement
  39. from sqlalchemy import Index
  40. from sqlalchemy import Table
  41. from sqlalchemy.engine import Connection
  42. from sqlalchemy.engine import Dialect
  43. from sqlalchemy.engine import Transaction
  44. from sqlalchemy.engine.reflection import Inspector
  45. from sqlalchemy.sql.base import ColumnCollection
  46. from sqlalchemy.sql.compiler import SQLCompiler
  47. from sqlalchemy.sql.dml import Insert
  48. from sqlalchemy.sql.elements import ColumnElement
  49. from sqlalchemy.sql.schema import Constraint
  50. from sqlalchemy.sql.schema import SchemaItem
  51. from sqlalchemy.sql.selectable import Select
  52. from sqlalchemy.sql.selectable import TableClause
  53. _CE = TypeVar("_CE", bound=Union["ColumnElement[Any]", "SchemaItem"])
  54. class _CompilerProtocol(Protocol):
  55. def __call__(self, element: Any, compiler: Any, **kw: Any) -> str:
  56. ...
  57. def _safe_int(value: str) -> Union[int, str]:
  58. try:
  59. return int(value)
  60. except:
  61. return value
  62. _vers = tuple(
  63. [_safe_int(x) for x in re.findall(r"(\d+|[abc]\d)", __version__)]
  64. )
  65. sqla_13 = _vers >= (1, 3)
  66. sqla_14 = _vers >= (1, 4)
  67. # https://docs.sqlalchemy.org/en/latest/changelog/changelog_14.html#change-0c6e0cc67dfe6fac5164720e57ef307d
  68. sqla_14_18 = _vers >= (1, 4, 18)
  69. sqla_14_26 = _vers >= (1, 4, 26)
  70. sqla_2 = _vers >= (2,)
  71. sqlalchemy_version = __version__
  72. try:
  73. from sqlalchemy.sql.naming import _NONE_NAME as _NONE_NAME # type: ignore[attr-defined] # noqa: E501
  74. except ImportError:
  75. from sqlalchemy.sql.elements import _NONE_NAME as _NONE_NAME # type: ignore # noqa: E501
  76. class _Unsupported:
  77. "Placeholder for unsupported SQLAlchemy classes"
  78. if TYPE_CHECKING:
  79. def compiles(
  80. element: Type[ClauseElement], *dialects: str
  81. ) -> Callable[[_CompilerProtocol], _CompilerProtocol]:
  82. ...
  83. else:
  84. from sqlalchemy.ext.compiler import compiles
  85. try:
  86. from sqlalchemy import Computed as Computed
  87. except ImportError:
  88. if not TYPE_CHECKING:
  89. class Computed(_Unsupported):
  90. pass
  91. has_computed = False
  92. has_computed_reflection = False
  93. else:
  94. has_computed = True
  95. has_computed_reflection = _vers >= (1, 3, 16)
  96. try:
  97. from sqlalchemy import Identity as Identity
  98. except ImportError:
  99. if not TYPE_CHECKING:
  100. class Identity(_Unsupported):
  101. pass
  102. has_identity = False
  103. else:
  104. identity_has_dialect_kwargs = issubclass(Identity, DialectKWArgs)
  105. def _get_identity_options_dict(
  106. identity: Union[Identity, schema.Sequence, None],
  107. dialect_kwargs: bool = False,
  108. ) -> Dict[str, Any]:
  109. if identity is None:
  110. return {}
  111. elif identity_has_dialect_kwargs:
  112. as_dict = identity._as_dict() # type: ignore
  113. if dialect_kwargs:
  114. assert isinstance(identity, DialectKWArgs)
  115. as_dict.update(identity.dialect_kwargs)
  116. else:
  117. as_dict = {}
  118. if isinstance(identity, Identity):
  119. # always=None means something different than always=False
  120. as_dict["always"] = identity.always
  121. if identity.on_null is not None:
  122. as_dict["on_null"] = identity.on_null
  123. # attributes common to Identity and Sequence
  124. attrs = (
  125. "start",
  126. "increment",
  127. "minvalue",
  128. "maxvalue",
  129. "nominvalue",
  130. "nomaxvalue",
  131. "cycle",
  132. "cache",
  133. "order",
  134. )
  135. as_dict.update(
  136. {
  137. key: getattr(identity, key, None)
  138. for key in attrs
  139. if getattr(identity, key, None) is not None
  140. }
  141. )
  142. return as_dict
  143. has_identity = True
  144. if sqla_2:
  145. from sqlalchemy.sql.base import _NoneName
  146. else:
  147. from sqlalchemy.util import symbol as _NoneName # type: ignore[assignment]
  148. _ConstraintName = Union[None, str, _NoneName]
  149. _ConstraintNameDefined = Union[str, _NoneName]
  150. def constraint_name_defined(
  151. name: _ConstraintName,
  152. ) -> TypeGuard[_ConstraintNameDefined]:
  153. return name is _NONE_NAME or isinstance(name, (str, _NoneName))
  154. def constraint_name_string(
  155. name: _ConstraintName,
  156. ) -> TypeGuard[str]:
  157. return isinstance(name, str)
  158. def constraint_name_or_none(
  159. name: _ConstraintName,
  160. ) -> Optional[str]:
  161. return name if constraint_name_string(name) else None
  162. AUTOINCREMENT_DEFAULT = "auto"
  163. @contextlib.contextmanager
  164. def _ensure_scope_for_ddl(
  165. connection: Optional[Connection],
  166. ) -> Iterator[None]:
  167. try:
  168. in_transaction = connection.in_transaction # type: ignore[union-attr]
  169. except AttributeError:
  170. # catch for MockConnection, None
  171. in_transaction = None
  172. pass
  173. # yield outside the catch
  174. if in_transaction is None:
  175. yield
  176. else:
  177. if not in_transaction():
  178. assert connection is not None
  179. with connection.begin():
  180. yield
  181. else:
  182. yield
  183. def url_render_as_string(url, hide_password=True):
  184. if sqla_14:
  185. return url.render_as_string(hide_password=hide_password)
  186. else:
  187. return url.__to_string__(hide_password=hide_password)
  188. def _safe_begin_connection_transaction(
  189. connection: Connection,
  190. ) -> Transaction:
  191. transaction = _get_connection_transaction(connection)
  192. if transaction:
  193. return transaction
  194. else:
  195. return connection.begin()
  196. def _safe_commit_connection_transaction(
  197. connection: Connection,
  198. ) -> None:
  199. transaction = _get_connection_transaction(connection)
  200. if transaction:
  201. transaction.commit()
  202. def _safe_rollback_connection_transaction(
  203. connection: Connection,
  204. ) -> None:
  205. transaction = _get_connection_transaction(connection)
  206. if transaction:
  207. transaction.rollback()
  208. def _get_connection_in_transaction(connection: Optional[Connection]) -> bool:
  209. try:
  210. in_transaction = connection.in_transaction # type: ignore
  211. except AttributeError:
  212. # catch for MockConnection
  213. return False
  214. else:
  215. return in_transaction()
  216. def _idx_table_bound_expressions(idx: Index) -> Iterable[ColumnElement[Any]]:
  217. return idx.expressions # type: ignore
  218. def _copy(schema_item: _CE, **kw) -> _CE:
  219. if hasattr(schema_item, "_copy"):
  220. return schema_item._copy(**kw)
  221. else:
  222. return schema_item.copy(**kw) # type: ignore[union-attr]
  223. def _get_connection_transaction(
  224. connection: Connection,
  225. ) -> Optional[Transaction]:
  226. if sqla_14:
  227. return connection.get_transaction()
  228. else:
  229. r = connection._root # type: ignore[attr-defined]
  230. return r._Connection__transaction
  231. def _create_url(*arg, **kw) -> url.URL:
  232. if hasattr(url.URL, "create"):
  233. return url.URL.create(*arg, **kw)
  234. else:
  235. return url.URL(*arg, **kw)
  236. def _connectable_has_table(
  237. connectable: Connection, tablename: str, schemaname: Union[str, None]
  238. ) -> bool:
  239. if sqla_14:
  240. return inspect(connectable).has_table(tablename, schemaname)
  241. else:
  242. return connectable.dialect.has_table(
  243. connectable, tablename, schemaname
  244. )
  245. def _exec_on_inspector(inspector, statement, **params):
  246. if sqla_14:
  247. with inspector._operation_context() as conn:
  248. return conn.execute(statement, params)
  249. else:
  250. return inspector.bind.execute(statement, params)
  251. def _nullability_might_be_unset(metadata_column):
  252. if not sqla_14:
  253. return metadata_column.nullable
  254. else:
  255. from sqlalchemy.sql import schema
  256. return (
  257. metadata_column._user_defined_nullable is schema.NULL_UNSPECIFIED
  258. )
  259. def _server_default_is_computed(*server_default) -> bool:
  260. if not has_computed:
  261. return False
  262. else:
  263. return any(isinstance(sd, Computed) for sd in server_default)
  264. def _server_default_is_identity(*server_default) -> bool:
  265. if not sqla_14:
  266. return False
  267. else:
  268. return any(isinstance(sd, Identity) for sd in server_default)
  269. def _table_for_constraint(constraint: Constraint) -> Table:
  270. if isinstance(constraint, ForeignKeyConstraint):
  271. table = constraint.parent
  272. assert table is not None
  273. return table # type: ignore[return-value]
  274. else:
  275. return constraint.table
  276. def _columns_for_constraint(constraint):
  277. if isinstance(constraint, ForeignKeyConstraint):
  278. return [fk.parent for fk in constraint.elements]
  279. elif isinstance(constraint, CheckConstraint):
  280. return _find_columns(constraint.sqltext)
  281. else:
  282. return list(constraint.columns)
  283. def _reflect_table(inspector: Inspector, table: Table) -> None:
  284. if sqla_14:
  285. return inspector.reflect_table(table, None)
  286. else:
  287. return inspector.reflecttable( # type: ignore[attr-defined]
  288. table, None
  289. )
  290. def _resolve_for_variant(type_, dialect):
  291. if _type_has_variants(type_):
  292. base_type, mapping = _get_variant_mapping(type_)
  293. return mapping.get(dialect.name, base_type)
  294. else:
  295. return type_
  296. if hasattr(sqltypes.TypeEngine, "_variant_mapping"):
  297. def _type_has_variants(type_):
  298. return bool(type_._variant_mapping)
  299. def _get_variant_mapping(type_):
  300. return type_, type_._variant_mapping
  301. else:
  302. def _type_has_variants(type_):
  303. return type(type_) is sqltypes.Variant
  304. def _get_variant_mapping(type_):
  305. return type_.impl, type_.mapping
  306. def _fk_spec(constraint: ForeignKeyConstraint) -> Any:
  307. if TYPE_CHECKING:
  308. assert constraint.columns is not None
  309. assert constraint.elements is not None
  310. assert isinstance(constraint.parent, Table)
  311. source_columns = [
  312. constraint.columns[key].name for key in constraint.column_keys
  313. ]
  314. source_table = constraint.parent.name
  315. source_schema = constraint.parent.schema
  316. target_schema = constraint.elements[0].column.table.schema
  317. target_table = constraint.elements[0].column.table.name
  318. target_columns = [element.column.name for element in constraint.elements]
  319. ondelete = constraint.ondelete
  320. onupdate = constraint.onupdate
  321. deferrable = constraint.deferrable
  322. initially = constraint.initially
  323. return (
  324. source_schema,
  325. source_table,
  326. source_columns,
  327. target_schema,
  328. target_table,
  329. target_columns,
  330. onupdate,
  331. ondelete,
  332. deferrable,
  333. initially,
  334. )
  335. def _fk_is_self_referential(constraint: ForeignKeyConstraint) -> bool:
  336. spec = constraint.elements[0]._get_colspec()
  337. tokens = spec.split(".")
  338. tokens.pop(-1) # colname
  339. tablekey = ".".join(tokens)
  340. assert constraint.parent is not None
  341. return tablekey == constraint.parent.key
  342. def _is_type_bound(constraint: Constraint) -> bool:
  343. # this deals with SQLAlchemy #3260, don't copy CHECK constraints
  344. # that will be generated by the type.
  345. # new feature added for #3260
  346. return constraint._type_bound
  347. def _find_columns(clause):
  348. """locate Column objects within the given expression."""
  349. cols: Set[ColumnElement[Any]] = set()
  350. traverse(clause, {}, {"column": cols.add})
  351. return cols
  352. def _remove_column_from_collection(
  353. collection: ColumnCollection, column: Union[Column[Any], ColumnClause[Any]]
  354. ) -> None:
  355. """remove a column from a ColumnCollection."""
  356. # workaround for older SQLAlchemy, remove the
  357. # same object that's present
  358. assert column.key is not None
  359. to_remove = collection[column.key]
  360. # SQLAlchemy 2.0 will use more ReadOnlyColumnCollection
  361. # (renamed from ImmutableColumnCollection)
  362. if hasattr(collection, "_immutable") or hasattr(collection, "_readonly"):
  363. collection._parent.remove(to_remove)
  364. else:
  365. collection.remove(to_remove)
  366. def _textual_index_column(
  367. table: Table, text_: Union[str, TextClause, ColumnElement[Any]]
  368. ) -> Union[ColumnElement[Any], Column[Any]]:
  369. """a workaround for the Index construct's severe lack of flexibility"""
  370. if isinstance(text_, str):
  371. c = Column(text_, sqltypes.NULLTYPE)
  372. table.append_column(c)
  373. return c
  374. elif isinstance(text_, TextClause):
  375. return _textual_index_element(table, text_)
  376. elif isinstance(text_, _textual_index_element):
  377. return _textual_index_column(table, text_.text)
  378. elif isinstance(text_, sql.ColumnElement):
  379. return _copy_expression(text_, table)
  380. else:
  381. raise ValueError("String or text() construct expected")
  382. def _copy_expression(expression: _CE, target_table: Table) -> _CE:
  383. def replace(col):
  384. if (
  385. isinstance(col, Column)
  386. and col.table is not None
  387. and col.table is not target_table
  388. ):
  389. if col.name in target_table.c:
  390. return target_table.c[col.name]
  391. else:
  392. c = _copy(col)
  393. target_table.append_column(c)
  394. return c
  395. else:
  396. return None
  397. return visitors.replacement_traverse( # type: ignore[call-overload]
  398. expression, {}, replace
  399. )
  400. class _textual_index_element(sql.ColumnElement):
  401. """Wrap around a sqlalchemy text() construct in such a way that
  402. we appear like a column-oriented SQL expression to an Index
  403. construct.
  404. The issue here is that currently the Postgresql dialect, the biggest
  405. recipient of functional indexes, keys all the index expressions to
  406. the corresponding column expressions when rendering CREATE INDEX,
  407. so the Index we create here needs to have a .columns collection that
  408. is the same length as the .expressions collection. Ultimately
  409. SQLAlchemy should support text() expressions in indexes.
  410. See SQLAlchemy issue 3174.
  411. """
  412. __visit_name__ = "_textual_idx_element"
  413. def __init__(self, table: Table, text: TextClause) -> None:
  414. self.table = table
  415. self.text = text
  416. self.key = text.text
  417. self.fake_column = schema.Column(self.text.text, sqltypes.NULLTYPE)
  418. table.append_column(self.fake_column)
  419. def get_children(self):
  420. return [self.fake_column]
  421. @compiles(_textual_index_element)
  422. def _render_textual_index_column(
  423. element: _textual_index_element, compiler: SQLCompiler, **kw
  424. ) -> str:
  425. return compiler.process(element.text, **kw)
  426. class _literal_bindparam(BindParameter):
  427. pass
  428. @compiles(_literal_bindparam)
  429. def _render_literal_bindparam(
  430. element: _literal_bindparam, compiler: SQLCompiler, **kw
  431. ) -> str:
  432. return compiler.render_literal_bindparam(element, **kw)
  433. def _column_kwargs(col: Column) -> Mapping:
  434. if sqla_13:
  435. return col.kwargs
  436. else:
  437. return {}
  438. def _get_constraint_final_name(
  439. constraint: Union[Index, Constraint], dialect: Optional[Dialect]
  440. ) -> Optional[str]:
  441. if constraint.name is None:
  442. return None
  443. assert dialect is not None
  444. if sqla_14:
  445. # for SQLAlchemy 1.4 we would like to have the option to expand
  446. # the use of "deferred" names for constraints as well as to have
  447. # some flexibility with "None" name and similar; make use of new
  448. # SQLAlchemy API to return what would be the final compiled form of
  449. # the name for this dialect.
  450. return dialect.identifier_preparer.format_constraint(
  451. constraint, _alembic_quote=False
  452. )
  453. else:
  454. # prior to SQLAlchemy 1.4, work around quoting logic to get at the
  455. # final compiled name without quotes.
  456. if hasattr(constraint.name, "quote"):
  457. # might be quoted_name, might be truncated_name, keep it the
  458. # same
  459. quoted_name_cls: type = type(constraint.name)
  460. else:
  461. quoted_name_cls = quoted_name
  462. new_name = quoted_name_cls(str(constraint.name), quote=False)
  463. constraint = constraint.__class__(name=new_name)
  464. if isinstance(constraint, schema.Index):
  465. # name should not be quoted.
  466. d = dialect.ddl_compiler(dialect, None) # type: ignore[arg-type]
  467. return d._prepared_index_name(constraint)
  468. else:
  469. # name should not be quoted.
  470. return dialect.identifier_preparer.format_constraint(constraint)
  471. def _constraint_is_named(
  472. constraint: Union[Constraint, Index], dialect: Optional[Dialect]
  473. ) -> bool:
  474. if sqla_14:
  475. if constraint.name is None:
  476. return False
  477. assert dialect is not None
  478. name = dialect.identifier_preparer.format_constraint(
  479. constraint, _alembic_quote=False
  480. )
  481. return name is not None
  482. else:
  483. return constraint.name is not None
  484. def _is_mariadb(mysql_dialect: Dialect) -> bool:
  485. if sqla_14:
  486. return mysql_dialect.is_mariadb # type: ignore[attr-defined]
  487. else:
  488. return bool(
  489. mysql_dialect.server_version_info
  490. and mysql_dialect._is_mariadb # type: ignore[attr-defined]
  491. )
  492. def _mariadb_normalized_version_info(mysql_dialect):
  493. return mysql_dialect._mariadb_normalized_version_info
  494. def _insert_inline(table: Union[TableClause, Table]) -> Insert:
  495. if sqla_14:
  496. return table.insert().inline()
  497. else:
  498. return table.insert(inline=True) # type: ignore[call-arg]
  499. if sqla_14:
  500. from sqlalchemy import create_mock_engine
  501. # weird mypy workaround
  502. from sqlalchemy import select as _sa_select
  503. _select = _sa_select
  504. else:
  505. from sqlalchemy import create_engine
  506. def create_mock_engine(url, executor, **kw): # type: ignore[misc]
  507. return create_engine(
  508. "postgresql://", strategy="mock", executor=executor
  509. )
  510. def _select(*columns, **kw) -> Select:
  511. return sql.select(list(columns), **kw) # type: ignore[call-overload]
  512. def is_expression_index(index: Index) -> bool:
  513. for expr in index.expressions:
  514. if is_expression(expr):
  515. return True
  516. return False
  517. def is_expression(expr: Any) -> bool:
  518. while isinstance(expr, UnaryExpression):
  519. expr = expr.element
  520. if not isinstance(expr, ColumnClause) or expr.is_literal:
  521. return True
  522. return False