typing.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585
  1. # util/typing.py
  2. # Copyright (C) 2022-2024 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: https://www.opensource.org/licenses/mit-license.php
  7. # mypy: allow-untyped-defs, allow-untyped-calls
  8. from __future__ import annotations
  9. import builtins
  10. import collections.abc as collections_abc
  11. import re
  12. import sys
  13. import typing
  14. from typing import Any
  15. from typing import Callable
  16. from typing import cast
  17. from typing import Dict
  18. from typing import ForwardRef
  19. from typing import Generic
  20. from typing import Iterable
  21. from typing import Mapping
  22. from typing import NewType
  23. from typing import NoReturn
  24. from typing import Optional
  25. from typing import overload
  26. from typing import Set
  27. from typing import Tuple
  28. from typing import Type
  29. from typing import TYPE_CHECKING
  30. from typing import TypeVar
  31. from typing import Union
  32. from . import compat
  33. if True: # zimports removes the tailing comments
  34. from typing_extensions import Annotated as Annotated # 3.8
  35. from typing_extensions import Concatenate as Concatenate # 3.10
  36. from typing_extensions import (
  37. dataclass_transform as dataclass_transform, # 3.11,
  38. )
  39. from typing_extensions import Final as Final # 3.8
  40. from typing_extensions import final as final # 3.8
  41. from typing_extensions import get_args as get_args # 3.10
  42. from typing_extensions import get_origin as get_origin # 3.10
  43. from typing_extensions import Literal as Literal # 3.8
  44. from typing_extensions import NotRequired as NotRequired # 3.11
  45. from typing_extensions import ParamSpec as ParamSpec # 3.10
  46. from typing_extensions import Protocol as Protocol # 3.8
  47. from typing_extensions import SupportsIndex as SupportsIndex # 3.8
  48. from typing_extensions import TypeAlias as TypeAlias # 3.10
  49. from typing_extensions import TypedDict as TypedDict # 3.8
  50. from typing_extensions import TypeGuard as TypeGuard # 3.10
  51. from typing_extensions import Self as Self # 3.11
  52. from typing_extensions import TypeAliasType as TypeAliasType # 3.12
  53. _T = TypeVar("_T", bound=Any)
  54. _KT = TypeVar("_KT")
  55. _KT_co = TypeVar("_KT_co", covariant=True)
  56. _KT_contra = TypeVar("_KT_contra", contravariant=True)
  57. _VT = TypeVar("_VT")
  58. _VT_co = TypeVar("_VT_co", covariant=True)
  59. if compat.py310:
  60. # why they took until py310 to put this in stdlib is beyond me,
  61. # I've been wanting it since py27
  62. from types import NoneType as NoneType
  63. else:
  64. NoneType = type(None) # type: ignore
  65. NoneFwd = ForwardRef("None")
  66. typing_get_args = get_args
  67. typing_get_origin = get_origin
  68. _AnnotationScanType = Union[
  69. Type[Any], str, ForwardRef, NewType, TypeAliasType, "GenericProtocol[Any]"
  70. ]
  71. class ArgsTypeProcotol(Protocol):
  72. """protocol for types that have ``__args__``
  73. there's no public interface for this AFAIK
  74. """
  75. __args__: Tuple[_AnnotationScanType, ...]
  76. class GenericProtocol(Protocol[_T]):
  77. """protocol for generic types.
  78. this since Python.typing _GenericAlias is private
  79. """
  80. __args__: Tuple[_AnnotationScanType, ...]
  81. __origin__: Type[_T]
  82. # Python's builtin _GenericAlias has this method, however builtins like
  83. # list, dict, etc. do not, even though they have ``__origin__`` and
  84. # ``__args__``
  85. #
  86. # def copy_with(self, params: Tuple[_AnnotationScanType, ...]) -> Type[_T]:
  87. # ...
  88. # copied from TypeShed, required in order to implement
  89. # MutableMapping.update()
  90. class SupportsKeysAndGetItem(Protocol[_KT, _VT_co]):
  91. def keys(self) -> Iterable[_KT]:
  92. ...
  93. def __getitem__(self, __k: _KT) -> _VT_co:
  94. ...
  95. # work around https://github.com/microsoft/pyright/issues/3025
  96. _LiteralStar = Literal["*"]
  97. def de_stringify_annotation(
  98. cls: Type[Any],
  99. annotation: _AnnotationScanType,
  100. originating_module: str,
  101. locals_: Mapping[str, Any],
  102. *,
  103. str_cleanup_fn: Optional[Callable[[str, str], str]] = None,
  104. include_generic: bool = False,
  105. _already_seen: Optional[Set[Any]] = None,
  106. ) -> Type[Any]:
  107. """Resolve annotations that may be string based into real objects.
  108. This is particularly important if a module defines "from __future__ import
  109. annotations", as everything inside of __annotations__ is a string. We want
  110. to at least have generic containers like ``Mapped``, ``Union``, ``List``,
  111. etc.
  112. """
  113. # looked at typing.get_type_hints(), looked at pydantic. We need much
  114. # less here, and we here try to not use any private typing internals
  115. # or construct ForwardRef objects which is documented as something
  116. # that should be avoided.
  117. original_annotation = annotation
  118. if is_fwd_ref(annotation):
  119. annotation = annotation.__forward_arg__
  120. if isinstance(annotation, str):
  121. if str_cleanup_fn:
  122. annotation = str_cleanup_fn(annotation, originating_module)
  123. annotation = eval_expression(
  124. annotation, originating_module, locals_=locals_
  125. )
  126. if (
  127. include_generic
  128. and is_generic(annotation)
  129. and not is_literal(annotation)
  130. ):
  131. if _already_seen is None:
  132. _already_seen = set()
  133. if annotation in _already_seen:
  134. # only occurs recursively. outermost return type
  135. # will always be Type.
  136. # the element here will be either ForwardRef or
  137. # Optional[ForwardRef]
  138. return original_annotation # type: ignore
  139. else:
  140. _already_seen.add(annotation)
  141. elements = tuple(
  142. de_stringify_annotation(
  143. cls,
  144. elem,
  145. originating_module,
  146. locals_,
  147. str_cleanup_fn=str_cleanup_fn,
  148. include_generic=include_generic,
  149. _already_seen=_already_seen,
  150. )
  151. for elem in annotation.__args__
  152. )
  153. return _copy_generic_annotation_with(annotation, elements)
  154. return annotation # type: ignore
  155. def _copy_generic_annotation_with(
  156. annotation: GenericProtocol[_T], elements: Tuple[_AnnotationScanType, ...]
  157. ) -> Type[_T]:
  158. if hasattr(annotation, "copy_with"):
  159. # List, Dict, etc. real generics
  160. return annotation.copy_with(elements) # type: ignore
  161. else:
  162. # Python builtins list, dict, etc.
  163. return annotation.__origin__[elements] # type: ignore
  164. def eval_expression(
  165. expression: str,
  166. module_name: str,
  167. *,
  168. locals_: Optional[Mapping[str, Any]] = None,
  169. ) -> Any:
  170. try:
  171. base_globals: Dict[str, Any] = sys.modules[module_name].__dict__
  172. except KeyError as ke:
  173. raise NameError(
  174. f"Module {module_name} isn't present in sys.modules; can't "
  175. f"evaluate expression {expression}"
  176. ) from ke
  177. try:
  178. annotation = eval(expression, base_globals, locals_)
  179. except Exception as err:
  180. raise NameError(
  181. f"Could not de-stringify annotation {expression!r}"
  182. ) from err
  183. else:
  184. return annotation
  185. def eval_name_only(
  186. name: str,
  187. module_name: str,
  188. *,
  189. locals_: Optional[Mapping[str, Any]] = None,
  190. ) -> Any:
  191. if "." in name:
  192. return eval_expression(name, module_name, locals_=locals_)
  193. try:
  194. base_globals: Dict[str, Any] = sys.modules[module_name].__dict__
  195. except KeyError as ke:
  196. raise NameError(
  197. f"Module {module_name} isn't present in sys.modules; can't "
  198. f"resolve name {name}"
  199. ) from ke
  200. # name only, just look in globals. eval() works perfectly fine here,
  201. # however we are seeking to have this be faster, as this occurs for
  202. # every Mapper[] keyword, etc. depending on configuration
  203. try:
  204. return base_globals[name]
  205. except KeyError as ke:
  206. # check in builtins as well to handle `list`, `set` or `dict`, etc.
  207. try:
  208. return builtins.__dict__[name]
  209. except KeyError:
  210. pass
  211. raise NameError(
  212. f"Could not locate name {name} in module {module_name}"
  213. ) from ke
  214. def resolve_name_to_real_class_name(name: str, module_name: str) -> str:
  215. try:
  216. obj = eval_name_only(name, module_name)
  217. except NameError:
  218. return name
  219. else:
  220. return getattr(obj, "__name__", name)
  221. def de_stringify_union_elements(
  222. cls: Type[Any],
  223. annotation: ArgsTypeProcotol,
  224. originating_module: str,
  225. locals_: Mapping[str, Any],
  226. *,
  227. str_cleanup_fn: Optional[Callable[[str, str], str]] = None,
  228. ) -> Type[Any]:
  229. return make_union_type(
  230. *[
  231. de_stringify_annotation(
  232. cls,
  233. anno,
  234. originating_module,
  235. {},
  236. str_cleanup_fn=str_cleanup_fn,
  237. )
  238. for anno in annotation.__args__
  239. ]
  240. )
  241. def is_pep593(type_: Optional[_AnnotationScanType]) -> bool:
  242. return type_ is not None and typing_get_origin(type_) is Annotated
  243. def is_non_string_iterable(obj: Any) -> TypeGuard[Iterable[Any]]:
  244. return isinstance(obj, collections_abc.Iterable) and not isinstance(
  245. obj, (str, bytes)
  246. )
  247. def is_literal(type_: _AnnotationScanType) -> bool:
  248. return get_origin(type_) is Literal
  249. def is_newtype(type_: Optional[_AnnotationScanType]) -> TypeGuard[NewType]:
  250. return hasattr(type_, "__supertype__")
  251. # doesn't work in 3.8, 3.7 as it passes a closure, not an
  252. # object instance
  253. # return isinstance(type_, NewType)
  254. def is_generic(type_: _AnnotationScanType) -> TypeGuard[GenericProtocol[Any]]:
  255. return hasattr(type_, "__args__") and hasattr(type_, "__origin__")
  256. def is_pep695(type_: _AnnotationScanType) -> TypeGuard[TypeAliasType]:
  257. return isinstance(type_, TypeAliasType)
  258. def flatten_newtype(type_: NewType) -> Type[Any]:
  259. super_type = type_.__supertype__
  260. while is_newtype(super_type):
  261. super_type = super_type.__supertype__
  262. return super_type
  263. def is_fwd_ref(
  264. type_: _AnnotationScanType, check_generic: bool = False
  265. ) -> TypeGuard[ForwardRef]:
  266. if isinstance(type_, ForwardRef):
  267. return True
  268. elif check_generic and is_generic(type_):
  269. return any(is_fwd_ref(arg, True) for arg in type_.__args__)
  270. else:
  271. return False
  272. @overload
  273. def de_optionalize_union_types(type_: str) -> str:
  274. ...
  275. @overload
  276. def de_optionalize_union_types(type_: Type[Any]) -> Type[Any]:
  277. ...
  278. @overload
  279. def de_optionalize_union_types(
  280. type_: _AnnotationScanType,
  281. ) -> _AnnotationScanType:
  282. ...
  283. def de_optionalize_union_types(
  284. type_: _AnnotationScanType,
  285. ) -> _AnnotationScanType:
  286. """Given a type, filter out ``Union`` types that include ``NoneType``
  287. to not include the ``NoneType``.
  288. """
  289. if is_fwd_ref(type_):
  290. return de_optionalize_fwd_ref_union_types(type_)
  291. elif is_optional(type_):
  292. typ = set(type_.__args__)
  293. typ.discard(NoneType)
  294. typ.discard(NoneFwd)
  295. return make_union_type(*typ)
  296. else:
  297. return type_
  298. def de_optionalize_fwd_ref_union_types(
  299. type_: ForwardRef,
  300. ) -> _AnnotationScanType:
  301. """return the non-optional type for Optional[], Union[None, ...], x|None,
  302. etc. without de-stringifying forward refs.
  303. unfortunately this seems to require lots of hardcoded heuristics
  304. """
  305. annotation = type_.__forward_arg__
  306. mm = re.match(r"^(.+?)\[(.+)\]$", annotation)
  307. if mm:
  308. if mm.group(1) == "Optional":
  309. return ForwardRef(mm.group(2))
  310. elif mm.group(1) == "Union":
  311. elements = re.split(r",\s*", mm.group(2))
  312. return make_union_type(
  313. *[ForwardRef(elem) for elem in elements if elem != "None"]
  314. )
  315. else:
  316. return type_
  317. pipe_tokens = re.split(r"\s*\|\s*", annotation)
  318. if "None" in pipe_tokens:
  319. return ForwardRef("|".join(p for p in pipe_tokens if p != "None"))
  320. return type_
  321. def make_union_type(*types: _AnnotationScanType) -> Type[Any]:
  322. """Make a Union type.
  323. This is needed by :func:`.de_optionalize_union_types` which removes
  324. ``NoneType`` from a ``Union``.
  325. """
  326. return cast(Any, Union).__getitem__(types) # type: ignore
  327. def expand_unions(
  328. type_: Type[Any], include_union: bool = False, discard_none: bool = False
  329. ) -> Tuple[Type[Any], ...]:
  330. """Return a type as a tuple of individual types, expanding for
  331. ``Union`` types."""
  332. if is_union(type_):
  333. typ = set(type_.__args__)
  334. if discard_none:
  335. typ.discard(NoneType)
  336. if include_union:
  337. return (type_,) + tuple(typ) # type: ignore
  338. else:
  339. return tuple(typ) # type: ignore
  340. else:
  341. return (type_,)
  342. def is_optional(type_: Any) -> TypeGuard[ArgsTypeProcotol]:
  343. return is_origin_of(
  344. type_,
  345. "Optional",
  346. "Union",
  347. "UnionType",
  348. )
  349. def is_optional_union(type_: Any) -> bool:
  350. return is_optional(type_) and NoneType in typing_get_args(type_)
  351. def is_union(type_: Any) -> TypeGuard[ArgsTypeProcotol]:
  352. return is_origin_of(type_, "Union")
  353. def is_origin_of_cls(
  354. type_: Any, class_obj: Union[Tuple[Type[Any], ...], Type[Any]]
  355. ) -> bool:
  356. """return True if the given type has an __origin__ that shares a base
  357. with the given class"""
  358. origin = typing_get_origin(type_)
  359. if origin is None:
  360. return False
  361. return isinstance(origin, type) and issubclass(origin, class_obj)
  362. def is_origin_of(
  363. type_: Any, *names: str, module: Optional[str] = None
  364. ) -> bool:
  365. """return True if the given type has an __origin__ with the given name
  366. and optional module."""
  367. origin = typing_get_origin(type_)
  368. if origin is None:
  369. return False
  370. return _get_type_name(origin) in names and (
  371. module is None or origin.__module__.startswith(module)
  372. )
  373. def _get_type_name(type_: Type[Any]) -> str:
  374. if compat.py310:
  375. return type_.__name__
  376. else:
  377. typ_name = getattr(type_, "__name__", None)
  378. if typ_name is None:
  379. typ_name = getattr(type_, "_name", None)
  380. return typ_name # type: ignore
  381. class DescriptorProto(Protocol):
  382. def __get__(self, instance: object, owner: Any) -> Any:
  383. ...
  384. def __set__(self, instance: Any, value: Any) -> None:
  385. ...
  386. def __delete__(self, instance: Any) -> None:
  387. ...
  388. _DESC = TypeVar("_DESC", bound=DescriptorProto)
  389. class DescriptorReference(Generic[_DESC]):
  390. """a descriptor that refers to a descriptor.
  391. used for cases where we need to have an instance variable referring to an
  392. object that is itself a descriptor, which typically confuses typing tools
  393. as they don't know when they should use ``__get__`` or not when referring
  394. to the descriptor assignment as an instance variable. See
  395. sqlalchemy.orm.interfaces.PropComparator.prop
  396. """
  397. if TYPE_CHECKING:
  398. def __get__(self, instance: object, owner: Any) -> _DESC:
  399. ...
  400. def __set__(self, instance: Any, value: _DESC) -> None:
  401. ...
  402. def __delete__(self, instance: Any) -> None:
  403. ...
  404. _DESC_co = TypeVar("_DESC_co", bound=DescriptorProto, covariant=True)
  405. class RODescriptorReference(Generic[_DESC_co]):
  406. """a descriptor that refers to a descriptor.
  407. same as :class:`.DescriptorReference` but is read-only, so that subclasses
  408. can define a subtype as the generically contained element
  409. """
  410. if TYPE_CHECKING:
  411. def __get__(self, instance: object, owner: Any) -> _DESC_co:
  412. ...
  413. def __set__(self, instance: Any, value: Any) -> NoReturn:
  414. ...
  415. def __delete__(self, instance: Any) -> NoReturn:
  416. ...
  417. _FN = TypeVar("_FN", bound=Optional[Callable[..., Any]])
  418. class CallableReference(Generic[_FN]):
  419. """a descriptor that refers to a callable.
  420. works around mypy's limitation of not allowing callables assigned
  421. as instance variables
  422. """
  423. if TYPE_CHECKING:
  424. def __get__(self, instance: object, owner: Any) -> _FN:
  425. ...
  426. def __set__(self, instance: Any, value: _FN) -> None:
  427. ...
  428. def __delete__(self, instance: Any) -> None:
  429. ...
  430. # $def ro_descriptor_reference(fn: Callable[])