annotation.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594
  1. # sql/annotation.py
  2. # Copyright (C) 2005-2024 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: https://www.opensource.org/licenses/mit-license.php
  7. """The :class:`.Annotated` class and related routines; creates hash-equivalent
  8. copies of SQL constructs which contain context-specific markers and
  9. associations.
  10. Note that the :class:`.Annotated` concept as implemented in this module is not
  11. related in any way to the pep-593 concept of "Annotated".
  12. """
  13. from __future__ import annotations
  14. import typing
  15. from typing import Any
  16. from typing import Callable
  17. from typing import cast
  18. from typing import Dict
  19. from typing import FrozenSet
  20. from typing import Mapping
  21. from typing import Optional
  22. from typing import overload
  23. from typing import Sequence
  24. from typing import Tuple
  25. from typing import Type
  26. from typing import TYPE_CHECKING
  27. from typing import TypeVar
  28. from . import operators
  29. from .cache_key import HasCacheKey
  30. from .visitors import anon_map
  31. from .visitors import ExternallyTraversible
  32. from .visitors import InternalTraversal
  33. from .. import util
  34. from ..util.typing import Literal
  35. from ..util.typing import Self
  36. if TYPE_CHECKING:
  37. from .base import _EntityNamespace
  38. from .visitors import _TraverseInternalsType
  39. _AnnotationDict = Mapping[str, Any]
  40. EMPTY_ANNOTATIONS: util.immutabledict[str, Any] = util.EMPTY_DICT
  41. class SupportsAnnotations(ExternallyTraversible):
  42. __slots__ = ()
  43. _annotations: util.immutabledict[str, Any] = EMPTY_ANNOTATIONS
  44. proxy_set: util.generic_fn_descriptor[FrozenSet[Any]]
  45. _is_immutable: bool
  46. def _annotate(self, values: _AnnotationDict) -> Self:
  47. raise NotImplementedError()
  48. @overload
  49. def _deannotate(
  50. self,
  51. values: Literal[None] = ...,
  52. clone: bool = ...,
  53. ) -> Self:
  54. ...
  55. @overload
  56. def _deannotate(
  57. self,
  58. values: Sequence[str] = ...,
  59. clone: bool = ...,
  60. ) -> SupportsAnnotations:
  61. ...
  62. def _deannotate(
  63. self,
  64. values: Optional[Sequence[str]] = None,
  65. clone: bool = False,
  66. ) -> SupportsAnnotations:
  67. raise NotImplementedError()
  68. @util.memoized_property
  69. def _annotations_cache_key(self) -> Tuple[Any, ...]:
  70. anon_map_ = anon_map()
  71. return self._gen_annotations_cache_key(anon_map_)
  72. def _gen_annotations_cache_key(
  73. self, anon_map: anon_map
  74. ) -> Tuple[Any, ...]:
  75. return (
  76. "_annotations",
  77. tuple(
  78. (
  79. key,
  80. value._gen_cache_key(anon_map, [])
  81. if isinstance(value, HasCacheKey)
  82. else value,
  83. )
  84. for key, value in [
  85. (key, self._annotations[key])
  86. for key in sorted(self._annotations)
  87. ]
  88. ),
  89. )
  90. class SupportsWrappingAnnotations(SupportsAnnotations):
  91. __slots__ = ()
  92. _constructor: Callable[..., SupportsWrappingAnnotations]
  93. if TYPE_CHECKING:
  94. @util.ro_non_memoized_property
  95. def entity_namespace(self) -> _EntityNamespace:
  96. ...
  97. def _annotate(self, values: _AnnotationDict) -> Self:
  98. """return a copy of this ClauseElement with annotations
  99. updated by the given dictionary.
  100. """
  101. return Annotated._as_annotated_instance(self, values) # type: ignore
  102. def _with_annotations(self, values: _AnnotationDict) -> Self:
  103. """return a copy of this ClauseElement with annotations
  104. replaced by the given dictionary.
  105. """
  106. return Annotated._as_annotated_instance(self, values) # type: ignore
  107. @overload
  108. def _deannotate(
  109. self,
  110. values: Literal[None] = ...,
  111. clone: bool = ...,
  112. ) -> Self:
  113. ...
  114. @overload
  115. def _deannotate(
  116. self,
  117. values: Sequence[str] = ...,
  118. clone: bool = ...,
  119. ) -> SupportsAnnotations:
  120. ...
  121. def _deannotate(
  122. self,
  123. values: Optional[Sequence[str]] = None,
  124. clone: bool = False,
  125. ) -> SupportsAnnotations:
  126. """return a copy of this :class:`_expression.ClauseElement`
  127. with annotations
  128. removed.
  129. :param values: optional tuple of individual values
  130. to remove.
  131. """
  132. if clone:
  133. s = self._clone()
  134. return s
  135. else:
  136. return self
  137. class SupportsCloneAnnotations(SupportsWrappingAnnotations):
  138. # SupportsCloneAnnotations extends from SupportsWrappingAnnotations
  139. # to support the structure of having the base ClauseElement
  140. # be a subclass of SupportsWrappingAnnotations. Any ClauseElement
  141. # subclass that wants to extend from SupportsCloneAnnotations
  142. # will inherently also be subclassing SupportsWrappingAnnotations, so
  143. # make that specific here.
  144. if not typing.TYPE_CHECKING:
  145. __slots__ = ()
  146. _clone_annotations_traverse_internals: _TraverseInternalsType = [
  147. ("_annotations", InternalTraversal.dp_annotations_key)
  148. ]
  149. def _annotate(self, values: _AnnotationDict) -> Self:
  150. """return a copy of this ClauseElement with annotations
  151. updated by the given dictionary.
  152. """
  153. new = self._clone()
  154. new._annotations = new._annotations.union(values)
  155. new.__dict__.pop("_annotations_cache_key", None)
  156. new.__dict__.pop("_generate_cache_key", None)
  157. return new
  158. def _with_annotations(self, values: _AnnotationDict) -> Self:
  159. """return a copy of this ClauseElement with annotations
  160. replaced by the given dictionary.
  161. """
  162. new = self._clone()
  163. new._annotations = util.immutabledict(values)
  164. new.__dict__.pop("_annotations_cache_key", None)
  165. new.__dict__.pop("_generate_cache_key", None)
  166. return new
  167. @overload
  168. def _deannotate(
  169. self,
  170. values: Literal[None] = ...,
  171. clone: bool = ...,
  172. ) -> Self:
  173. ...
  174. @overload
  175. def _deannotate(
  176. self,
  177. values: Sequence[str] = ...,
  178. clone: bool = ...,
  179. ) -> SupportsAnnotations:
  180. ...
  181. def _deannotate(
  182. self,
  183. values: Optional[Sequence[str]] = None,
  184. clone: bool = False,
  185. ) -> SupportsAnnotations:
  186. """return a copy of this :class:`_expression.ClauseElement`
  187. with annotations
  188. removed.
  189. :param values: optional tuple of individual values
  190. to remove.
  191. """
  192. if clone or self._annotations:
  193. # clone is used when we are also copying
  194. # the expression for a deep deannotation
  195. new = self._clone()
  196. new._annotations = util.immutabledict()
  197. new.__dict__.pop("_annotations_cache_key", None)
  198. return new
  199. else:
  200. return self
  201. class Annotated(SupportsAnnotations):
  202. """clones a SupportsAnnotations and applies an 'annotations' dictionary.
  203. Unlike regular clones, this clone also mimics __hash__() and
  204. __eq__() of the original element so that it takes its place
  205. in hashed collections.
  206. A reference to the original element is maintained, for the important
  207. reason of keeping its hash value current. When GC'ed, the
  208. hash value may be reused, causing conflicts.
  209. .. note:: The rationale for Annotated producing a brand new class,
  210. rather than placing the functionality directly within ClauseElement,
  211. is **performance**. The __hash__() method is absent on plain
  212. ClauseElement which leads to significantly reduced function call
  213. overhead, as the use of sets and dictionaries against ClauseElement
  214. objects is prevalent, but most are not "annotated".
  215. """
  216. _is_column_operators = False
  217. @classmethod
  218. def _as_annotated_instance(
  219. cls, element: SupportsWrappingAnnotations, values: _AnnotationDict
  220. ) -> Annotated:
  221. try:
  222. cls = annotated_classes[element.__class__]
  223. except KeyError:
  224. cls = _new_annotation_type(element.__class__, cls)
  225. return cls(element, values)
  226. _annotations: util.immutabledict[str, Any]
  227. __element: SupportsWrappingAnnotations
  228. _hash: int
  229. def __new__(cls: Type[Self], *args: Any) -> Self:
  230. return object.__new__(cls)
  231. def __init__(
  232. self, element: SupportsWrappingAnnotations, values: _AnnotationDict
  233. ):
  234. self.__dict__ = element.__dict__.copy()
  235. self.__dict__.pop("_annotations_cache_key", None)
  236. self.__dict__.pop("_generate_cache_key", None)
  237. self.__element = element
  238. self._annotations = util.immutabledict(values)
  239. self._hash = hash(element)
  240. def _annotate(self, values: _AnnotationDict) -> Self:
  241. _values = self._annotations.union(values)
  242. new = self._with_annotations(_values)
  243. return new
  244. def _with_annotations(self, values: _AnnotationDict) -> Self:
  245. clone = self.__class__.__new__(self.__class__)
  246. clone.__dict__ = self.__dict__.copy()
  247. clone.__dict__.pop("_annotations_cache_key", None)
  248. clone.__dict__.pop("_generate_cache_key", None)
  249. clone._annotations = util.immutabledict(values)
  250. return clone
  251. @overload
  252. def _deannotate(
  253. self,
  254. values: Literal[None] = ...,
  255. clone: bool = ...,
  256. ) -> Self:
  257. ...
  258. @overload
  259. def _deannotate(
  260. self,
  261. values: Sequence[str] = ...,
  262. clone: bool = ...,
  263. ) -> Annotated:
  264. ...
  265. def _deannotate(
  266. self,
  267. values: Optional[Sequence[str]] = None,
  268. clone: bool = True,
  269. ) -> SupportsAnnotations:
  270. if values is None:
  271. return self.__element
  272. else:
  273. return self._with_annotations(
  274. util.immutabledict(
  275. {
  276. key: value
  277. for key, value in self._annotations.items()
  278. if key not in values
  279. }
  280. )
  281. )
  282. if not typing.TYPE_CHECKING:
  283. # manually proxy some methods that need extra attention
  284. def _compiler_dispatch(self, visitor: Any, **kw: Any) -> Any:
  285. return self.__element.__class__._compiler_dispatch(
  286. self, visitor, **kw
  287. )
  288. @property
  289. def _constructor(self):
  290. return self.__element._constructor
  291. def _clone(self, **kw: Any) -> Self:
  292. clone = self.__element._clone(**kw)
  293. if clone is self.__element:
  294. # detect immutable, don't change anything
  295. return self
  296. else:
  297. # update the clone with any changes that have occurred
  298. # to this object's __dict__.
  299. clone.__dict__.update(self.__dict__)
  300. return self.__class__(clone, self._annotations)
  301. def __reduce__(self) -> Tuple[Type[Annotated], Tuple[Any, ...]]:
  302. return self.__class__, (self.__element, self._annotations)
  303. def __hash__(self) -> int:
  304. return self._hash
  305. def __eq__(self, other: Any) -> bool:
  306. if self._is_column_operators:
  307. return self.__element.__class__.__eq__(self, other)
  308. else:
  309. return hash(other) == hash(self)
  310. @util.ro_non_memoized_property
  311. def entity_namespace(self) -> _EntityNamespace:
  312. if "entity_namespace" in self._annotations:
  313. return cast(
  314. SupportsWrappingAnnotations,
  315. self._annotations["entity_namespace"],
  316. ).entity_namespace
  317. else:
  318. return self.__element.entity_namespace
  319. # hard-generate Annotated subclasses. this technique
  320. # is used instead of on-the-fly types (i.e. type.__new__())
  321. # so that the resulting objects are pickleable; additionally, other
  322. # decisions can be made up front about the type of object being annotated
  323. # just once per class rather than per-instance.
  324. annotated_classes: Dict[
  325. Type[SupportsWrappingAnnotations], Type[Annotated]
  326. ] = {}
  327. _SA = TypeVar("_SA", bound="SupportsAnnotations")
  328. def _safe_annotate(to_annotate: _SA, annotations: _AnnotationDict) -> _SA:
  329. try:
  330. _annotate = to_annotate._annotate
  331. except AttributeError:
  332. # skip objects that don't actually have an `_annotate`
  333. # attribute, namely QueryableAttribute inside of a join
  334. # condition
  335. return to_annotate
  336. else:
  337. return _annotate(annotations)
  338. def _deep_annotate(
  339. element: _SA,
  340. annotations: _AnnotationDict,
  341. exclude: Optional[Sequence[SupportsAnnotations]] = None,
  342. *,
  343. detect_subquery_cols: bool = False,
  344. ind_cols_on_fromclause: bool = False,
  345. annotate_callable: Optional[
  346. Callable[[SupportsAnnotations, _AnnotationDict], SupportsAnnotations]
  347. ] = None,
  348. ) -> _SA:
  349. """Deep copy the given ClauseElement, annotating each element
  350. with the given annotations dictionary.
  351. Elements within the exclude collection will be cloned but not annotated.
  352. """
  353. # annotated objects hack the __hash__() method so if we want to
  354. # uniquely process them we have to use id()
  355. cloned_ids: Dict[int, SupportsAnnotations] = {}
  356. def clone(elem: SupportsAnnotations, **kw: Any) -> SupportsAnnotations:
  357. # ind_cols_on_fromclause means make sure an AnnotatedFromClause
  358. # has its own .c collection independent of that which its proxying.
  359. # this is used specifically by orm.LoaderCriteriaOption to break
  360. # a reference cycle that it's otherwise prone to building,
  361. # see test_relationship_criteria->
  362. # test_loader_criteria_subquery_w_same_entity. logic here was
  363. # changed for #8796 and made explicit; previously it occurred
  364. # by accident
  365. kw["detect_subquery_cols"] = detect_subquery_cols
  366. id_ = id(elem)
  367. if id_ in cloned_ids:
  368. return cloned_ids[id_]
  369. if (
  370. exclude
  371. and hasattr(elem, "proxy_set")
  372. and elem.proxy_set.intersection(exclude)
  373. ):
  374. newelem = elem._clone(clone=clone, **kw)
  375. elif annotations != elem._annotations:
  376. if detect_subquery_cols and elem._is_immutable:
  377. to_annotate = elem._clone(clone=clone, **kw)
  378. else:
  379. to_annotate = elem
  380. if annotate_callable:
  381. newelem = annotate_callable(to_annotate, annotations)
  382. else:
  383. newelem = _safe_annotate(to_annotate, annotations)
  384. else:
  385. newelem = elem
  386. newelem._copy_internals(
  387. clone=clone, ind_cols_on_fromclause=ind_cols_on_fromclause
  388. )
  389. cloned_ids[id_] = newelem
  390. return newelem
  391. if element is not None:
  392. element = cast(_SA, clone(element))
  393. clone = None # type: ignore # remove gc cycles
  394. return element
  395. @overload
  396. def _deep_deannotate(
  397. element: Literal[None], values: Optional[Sequence[str]] = None
  398. ) -> Literal[None]:
  399. ...
  400. @overload
  401. def _deep_deannotate(
  402. element: _SA, values: Optional[Sequence[str]] = None
  403. ) -> _SA:
  404. ...
  405. def _deep_deannotate(
  406. element: Optional[_SA], values: Optional[Sequence[str]] = None
  407. ) -> Optional[_SA]:
  408. """Deep copy the given element, removing annotations."""
  409. cloned: Dict[Any, SupportsAnnotations] = {}
  410. def clone(elem: SupportsAnnotations, **kw: Any) -> SupportsAnnotations:
  411. key: Any
  412. if values:
  413. key = id(elem)
  414. else:
  415. key = elem
  416. if key not in cloned:
  417. newelem = elem._deannotate(values=values, clone=True)
  418. newelem._copy_internals(clone=clone)
  419. cloned[key] = newelem
  420. return newelem
  421. else:
  422. return cloned[key]
  423. if element is not None:
  424. element = cast(_SA, clone(element))
  425. clone = None # type: ignore # remove gc cycles
  426. return element
  427. def _shallow_annotate(element: _SA, annotations: _AnnotationDict) -> _SA:
  428. """Annotate the given ClauseElement and copy its internals so that
  429. internal objects refer to the new annotated object.
  430. Basically used to apply a "don't traverse" annotation to a
  431. selectable, without digging throughout the whole
  432. structure wasting time.
  433. """
  434. element = element._annotate(annotations)
  435. element._copy_internals()
  436. return element
  437. def _new_annotation_type(
  438. cls: Type[SupportsWrappingAnnotations], base_cls: Type[Annotated]
  439. ) -> Type[Annotated]:
  440. """Generates a new class that subclasses Annotated and proxies a given
  441. element type.
  442. """
  443. if issubclass(cls, Annotated):
  444. return cls
  445. elif cls in annotated_classes:
  446. return annotated_classes[cls]
  447. for super_ in cls.__mro__:
  448. # check if an Annotated subclass more specific than
  449. # the given base_cls is already registered, such
  450. # as AnnotatedColumnElement.
  451. if super_ in annotated_classes:
  452. base_cls = annotated_classes[super_]
  453. break
  454. annotated_classes[cls] = anno_cls = cast(
  455. Type[Annotated],
  456. type("Annotated%s" % cls.__name__, (base_cls, cls), {}),
  457. )
  458. globals()["Annotated%s" % cls.__name__] = anno_cls
  459. if "_traverse_internals" in cls.__dict__:
  460. anno_cls._traverse_internals = list(cls._traverse_internals) + [
  461. ("_annotations", InternalTraversal.dp_annotations_key)
  462. ]
  463. elif cls.__dict__.get("inherit_cache", False):
  464. anno_cls._traverse_internals = list(cls._traverse_internals) + [
  465. ("_annotations", InternalTraversal.dp_annotations_key)
  466. ]
  467. # some classes include this even if they have traverse_internals
  468. # e.g. BindParameter, add it if present.
  469. if cls.__dict__.get("inherit_cache", False):
  470. anno_cls.inherit_cache = True # type: ignore
  471. elif "inherit_cache" in cls.__dict__:
  472. anno_cls.inherit_cache = cls.__dict__["inherit_cache"] # type: ignore
  473. anno_cls._is_column_operators = issubclass(cls, operators.ColumnOperators)
  474. return anno_cls
  475. def _prepare_annotations(
  476. target_hierarchy: Type[SupportsWrappingAnnotations],
  477. base_cls: Type[Annotated],
  478. ) -> None:
  479. for cls in util.walk_subclasses(target_hierarchy):
  480. _new_annotation_type(cls, base_cls)