typing.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603
  1. import sys
  2. import typing
  3. from collections.abc import Callable
  4. from os import PathLike
  5. from typing import ( # type: ignore
  6. TYPE_CHECKING,
  7. AbstractSet,
  8. Any,
  9. Callable as TypingCallable,
  10. ClassVar,
  11. Dict,
  12. ForwardRef,
  13. Generator,
  14. Iterable,
  15. List,
  16. Mapping,
  17. NewType,
  18. Optional,
  19. Sequence,
  20. Set,
  21. Tuple,
  22. Type,
  23. TypeVar,
  24. Union,
  25. _eval_type,
  26. cast,
  27. get_type_hints,
  28. )
  29. from typing_extensions import (
  30. Annotated,
  31. Final,
  32. Literal,
  33. NotRequired as TypedDictNotRequired,
  34. Required as TypedDictRequired,
  35. )
  36. try:
  37. from typing import _TypingBase as typing_base # type: ignore
  38. except ImportError:
  39. from typing import _Final as typing_base # type: ignore
  40. try:
  41. from typing import GenericAlias as TypingGenericAlias # type: ignore
  42. except ImportError:
  43. # python < 3.9 does not have GenericAlias (list[int], tuple[str, ...] and so on)
  44. TypingGenericAlias = ()
  45. try:
  46. from types import UnionType as TypesUnionType # type: ignore
  47. except ImportError:
  48. # python < 3.10 does not have UnionType (str | int, byte | bool and so on)
  49. TypesUnionType = ()
  50. if sys.version_info < (3, 9):
  51. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  52. return type_._evaluate(globalns, localns)
  53. else:
  54. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  55. # Even though it is the right signature for python 3.9, mypy complains with
  56. # `error: Too many arguments for "_evaluate" of "ForwardRef"` hence the cast...
  57. return cast(Any, type_)._evaluate(globalns, localns, set())
  58. if sys.version_info < (3, 9):
  59. # Ensure we always get all the whole `Annotated` hint, not just the annotated type.
  60. # For 3.7 to 3.8, `get_type_hints` doesn't recognize `typing_extensions.Annotated`,
  61. # so it already returns the full annotation
  62. get_all_type_hints = get_type_hints
  63. else:
  64. def get_all_type_hints(obj: Any, globalns: Any = None, localns: Any = None) -> Any:
  65. return get_type_hints(obj, globalns, localns, include_extras=True)
  66. _T = TypeVar('_T')
  67. AnyCallable = TypingCallable[..., Any]
  68. NoArgAnyCallable = TypingCallable[[], Any]
  69. # workaround for https://github.com/python/mypy/issues/9496
  70. AnyArgTCallable = TypingCallable[..., _T]
  71. # Annotated[...] is implemented by returning an instance of one of these classes, depending on
  72. # python/typing_extensions version.
  73. AnnotatedTypeNames = {'AnnotatedMeta', '_AnnotatedAlias'}
  74. LITERAL_TYPES: Set[Any] = {Literal}
  75. if hasattr(typing, 'Literal'):
  76. LITERAL_TYPES.add(typing.Literal)
  77. if sys.version_info < (3, 8):
  78. def get_origin(t: Type[Any]) -> Optional[Type[Any]]:
  79. if type(t).__name__ in AnnotatedTypeNames:
  80. # weirdly this is a runtime requirement, as well as for mypy
  81. return cast(Type[Any], Annotated)
  82. return getattr(t, '__origin__', None)
  83. else:
  84. from typing import get_origin as _typing_get_origin
  85. def get_origin(tp: Type[Any]) -> Optional[Type[Any]]:
  86. """
  87. We can't directly use `typing.get_origin` since we need a fallback to support
  88. custom generic classes like `ConstrainedList`
  89. It should be useless once https://github.com/cython/cython/issues/3537 is
  90. solved and https://github.com/pydantic/pydantic/pull/1753 is merged.
  91. """
  92. if type(tp).__name__ in AnnotatedTypeNames:
  93. return cast(Type[Any], Annotated) # mypy complains about _SpecialForm
  94. return _typing_get_origin(tp) or getattr(tp, '__origin__', None)
  95. if sys.version_info < (3, 8):
  96. from typing import _GenericAlias
  97. def get_args(t: Type[Any]) -> Tuple[Any, ...]:
  98. """Compatibility version of get_args for python 3.7.
  99. Mostly compatible with the python 3.8 `typing` module version
  100. and able to handle almost all use cases.
  101. """
  102. if type(t).__name__ in AnnotatedTypeNames:
  103. return t.__args__ + t.__metadata__
  104. if isinstance(t, _GenericAlias):
  105. res = t.__args__
  106. if t.__origin__ is Callable and res and res[0] is not Ellipsis:
  107. res = (list(res[:-1]), res[-1])
  108. return res
  109. return getattr(t, '__args__', ())
  110. else:
  111. from typing import get_args as _typing_get_args
  112. def _generic_get_args(tp: Type[Any]) -> Tuple[Any, ...]:
  113. """
  114. In python 3.9, `typing.Dict`, `typing.List`, ...
  115. do have an empty `__args__` by default (instead of the generic ~T for example).
  116. In order to still support `Dict` for example and consider it as `Dict[Any, Any]`,
  117. we retrieve the `_nparams` value that tells us how many parameters it needs.
  118. """
  119. if hasattr(tp, '_nparams'):
  120. return (Any,) * tp._nparams
  121. # Special case for `tuple[()]`, which used to return ((),) with `typing.Tuple`
  122. # in python 3.10- but now returns () for `tuple` and `Tuple`.
  123. # This will probably be clarified in pydantic v2
  124. try:
  125. if tp == Tuple[()] or sys.version_info >= (3, 9) and tp == tuple[()]: # type: ignore[misc]
  126. return ((),)
  127. # there is a TypeError when compiled with cython
  128. except TypeError: # pragma: no cover
  129. pass
  130. return ()
  131. def get_args(tp: Type[Any]) -> Tuple[Any, ...]:
  132. """Get type arguments with all substitutions performed.
  133. For unions, basic simplifications used by Union constructor are performed.
  134. Examples::
  135. get_args(Dict[str, int]) == (str, int)
  136. get_args(int) == ()
  137. get_args(Union[int, Union[T, int], str][int]) == (int, str)
  138. get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int])
  139. get_args(Callable[[], T][int]) == ([], int)
  140. """
  141. if type(tp).__name__ in AnnotatedTypeNames:
  142. return tp.__args__ + tp.__metadata__
  143. # the fallback is needed for the same reasons as `get_origin` (see above)
  144. return _typing_get_args(tp) or getattr(tp, '__args__', ()) or _generic_get_args(tp)
  145. if sys.version_info < (3, 9):
  146. def convert_generics(tp: Type[Any]) -> Type[Any]:
  147. """Python 3.9 and older only supports generics from `typing` module.
  148. They convert strings to ForwardRef automatically.
  149. Examples::
  150. typing.List['Hero'] == typing.List[ForwardRef('Hero')]
  151. """
  152. return tp
  153. else:
  154. from typing import _UnionGenericAlias # type: ignore
  155. from typing_extensions import _AnnotatedAlias
  156. def convert_generics(tp: Type[Any]) -> Type[Any]:
  157. """
  158. Recursively searches for `str` type hints and replaces them with ForwardRef.
  159. Examples::
  160. convert_generics(list['Hero']) == list[ForwardRef('Hero')]
  161. convert_generics(dict['Hero', 'Team']) == dict[ForwardRef('Hero'), ForwardRef('Team')]
  162. convert_generics(typing.Dict['Hero', 'Team']) == typing.Dict[ForwardRef('Hero'), ForwardRef('Team')]
  163. convert_generics(list[str | 'Hero'] | int) == list[str | ForwardRef('Hero')] | int
  164. """
  165. origin = get_origin(tp)
  166. if not origin or not hasattr(tp, '__args__'):
  167. return tp
  168. args = get_args(tp)
  169. # typing.Annotated needs special treatment
  170. if origin is Annotated:
  171. return _AnnotatedAlias(convert_generics(args[0]), args[1:])
  172. # recursively replace `str` instances inside of `GenericAlias` with `ForwardRef(arg)`
  173. converted = tuple(
  174. ForwardRef(arg) if isinstance(arg, str) and isinstance(tp, TypingGenericAlias) else convert_generics(arg)
  175. for arg in args
  176. )
  177. if converted == args:
  178. return tp
  179. elif isinstance(tp, TypingGenericAlias):
  180. return TypingGenericAlias(origin, converted)
  181. elif isinstance(tp, TypesUnionType):
  182. # recreate types.UnionType (PEP604, Python >= 3.10)
  183. return _UnionGenericAlias(origin, converted)
  184. else:
  185. try:
  186. setattr(tp, '__args__', converted)
  187. except AttributeError:
  188. pass
  189. return tp
  190. if sys.version_info < (3, 10):
  191. def is_union(tp: Optional[Type[Any]]) -> bool:
  192. return tp is Union
  193. WithArgsTypes = (TypingGenericAlias,)
  194. else:
  195. import types
  196. import typing
  197. def is_union(tp: Optional[Type[Any]]) -> bool:
  198. return tp is Union or tp is types.UnionType # noqa: E721
  199. WithArgsTypes = (typing._GenericAlias, types.GenericAlias, types.UnionType)
  200. StrPath = Union[str, PathLike]
  201. if TYPE_CHECKING:
  202. from .fields import ModelField
  203. TupleGenerator = Generator[Tuple[str, Any], None, None]
  204. DictStrAny = Dict[str, Any]
  205. DictAny = Dict[Any, Any]
  206. SetStr = Set[str]
  207. ListStr = List[str]
  208. IntStr = Union[int, str]
  209. AbstractSetIntStr = AbstractSet[IntStr]
  210. DictIntStrAny = Dict[IntStr, Any]
  211. MappingIntStrAny = Mapping[IntStr, Any]
  212. CallableGenerator = Generator[AnyCallable, None, None]
  213. ReprArgs = Sequence[Tuple[Optional[str], Any]]
  214. MYPY = False
  215. if MYPY:
  216. AnyClassMethod = classmethod[Any]
  217. else:
  218. # classmethod[TargetType, CallableParamSpecType, CallableReturnType]
  219. AnyClassMethod = classmethod[Any, Any, Any]
  220. __all__ = (
  221. 'AnyCallable',
  222. 'NoArgAnyCallable',
  223. 'NoneType',
  224. 'is_none_type',
  225. 'display_as_type',
  226. 'resolve_annotations',
  227. 'is_callable_type',
  228. 'is_literal_type',
  229. 'all_literal_values',
  230. 'is_namedtuple',
  231. 'is_typeddict',
  232. 'is_typeddict_special',
  233. 'is_new_type',
  234. 'new_type_supertype',
  235. 'is_classvar',
  236. 'is_finalvar',
  237. 'update_field_forward_refs',
  238. 'update_model_forward_refs',
  239. 'TupleGenerator',
  240. 'DictStrAny',
  241. 'DictAny',
  242. 'SetStr',
  243. 'ListStr',
  244. 'IntStr',
  245. 'AbstractSetIntStr',
  246. 'DictIntStrAny',
  247. 'CallableGenerator',
  248. 'ReprArgs',
  249. 'AnyClassMethod',
  250. 'CallableGenerator',
  251. 'WithArgsTypes',
  252. 'get_args',
  253. 'get_origin',
  254. 'get_sub_types',
  255. 'typing_base',
  256. 'get_all_type_hints',
  257. 'is_union',
  258. 'StrPath',
  259. 'MappingIntStrAny',
  260. )
  261. NoneType = None.__class__
  262. NONE_TYPES: Tuple[Any, Any, Any] = (None, NoneType, Literal[None])
  263. if sys.version_info < (3, 8):
  264. # Even though this implementation is slower, we need it for python 3.7:
  265. # In python 3.7 "Literal" is not a builtin type and uses a different
  266. # mechanism.
  267. # for this reason `Literal[None] is Literal[None]` evaluates to `False`,
  268. # breaking the faster implementation used for the other python versions.
  269. def is_none_type(type_: Any) -> bool:
  270. return type_ in NONE_TYPES
  271. elif sys.version_info[:2] == (3, 8):
  272. def is_none_type(type_: Any) -> bool:
  273. for none_type in NONE_TYPES:
  274. if type_ is none_type:
  275. return True
  276. # With python 3.8, specifically 3.8.10, Literal "is" check sare very flakey
  277. # can change on very subtle changes like use of types in other modules,
  278. # hopefully this check avoids that issue.
  279. if is_literal_type(type_): # pragma: no cover
  280. return all_literal_values(type_) == (None,)
  281. return False
  282. else:
  283. def is_none_type(type_: Any) -> bool:
  284. return type_ in NONE_TYPES
  285. def display_as_type(v: Type[Any]) -> str:
  286. if not isinstance(v, typing_base) and not isinstance(v, WithArgsTypes) and not isinstance(v, type):
  287. v = v.__class__
  288. if is_union(get_origin(v)):
  289. return f'Union[{", ".join(map(display_as_type, get_args(v)))}]'
  290. if isinstance(v, WithArgsTypes):
  291. # Generic alias are constructs like `list[int]`
  292. return str(v).replace('typing.', '')
  293. try:
  294. return v.__name__
  295. except AttributeError:
  296. # happens with typing objects
  297. return str(v).replace('typing.', '')
  298. def resolve_annotations(raw_annotations: Dict[str, Type[Any]], module_name: Optional[str]) -> Dict[str, Type[Any]]:
  299. """
  300. Partially taken from typing.get_type_hints.
  301. Resolve string or ForwardRef annotations into type objects if possible.
  302. """
  303. base_globals: Optional[Dict[str, Any]] = None
  304. if module_name:
  305. try:
  306. module = sys.modules[module_name]
  307. except KeyError:
  308. # happens occasionally, see https://github.com/pydantic/pydantic/issues/2363
  309. pass
  310. else:
  311. base_globals = module.__dict__
  312. annotations = {}
  313. for name, value in raw_annotations.items():
  314. if isinstance(value, str):
  315. if (3, 10) > sys.version_info >= (3, 9, 8) or sys.version_info >= (3, 10, 1):
  316. value = ForwardRef(value, is_argument=False, is_class=True)
  317. else:
  318. value = ForwardRef(value, is_argument=False)
  319. try:
  320. value = _eval_type(value, base_globals, None)
  321. except NameError:
  322. # this is ok, it can be fixed with update_forward_refs
  323. pass
  324. annotations[name] = value
  325. return annotations
  326. def is_callable_type(type_: Type[Any]) -> bool:
  327. return type_ is Callable or get_origin(type_) is Callable
  328. def is_literal_type(type_: Type[Any]) -> bool:
  329. return Literal is not None and get_origin(type_) in LITERAL_TYPES
  330. def literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
  331. return get_args(type_)
  332. def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
  333. """
  334. This method is used to retrieve all Literal values as
  335. Literal can be used recursively (see https://www.python.org/dev/peps/pep-0586)
  336. e.g. `Literal[Literal[Literal[1, 2, 3], "foo"], 5, None]`
  337. """
  338. if not is_literal_type(type_):
  339. return (type_,)
  340. values = literal_values(type_)
  341. return tuple(x for value in values for x in all_literal_values(value))
  342. def is_namedtuple(type_: Type[Any]) -> bool:
  343. """
  344. Check if a given class is a named tuple.
  345. It can be either a `typing.NamedTuple` or `collections.namedtuple`
  346. """
  347. from .utils import lenient_issubclass
  348. return lenient_issubclass(type_, tuple) and hasattr(type_, '_fields')
  349. def is_typeddict(type_: Type[Any]) -> bool:
  350. """
  351. Check if a given class is a typed dict (from `typing` or `typing_extensions`)
  352. In 3.10, there will be a public method (https://docs.python.org/3.10/library/typing.html#typing.is_typeddict)
  353. """
  354. from .utils import lenient_issubclass
  355. return lenient_issubclass(type_, dict) and hasattr(type_, '__total__')
  356. def _check_typeddict_special(type_: Any) -> bool:
  357. return type_ is TypedDictRequired or type_ is TypedDictNotRequired
  358. def is_typeddict_special(type_: Any) -> bool:
  359. """
  360. Check if type is a TypedDict special form (Required or NotRequired).
  361. """
  362. return _check_typeddict_special(type_) or _check_typeddict_special(get_origin(type_))
  363. test_type = NewType('test_type', str)
  364. def is_new_type(type_: Type[Any]) -> bool:
  365. """
  366. Check whether type_ was created using typing.NewType
  367. """
  368. return isinstance(type_, test_type.__class__) and hasattr(type_, '__supertype__') # type: ignore
  369. def new_type_supertype(type_: Type[Any]) -> Type[Any]:
  370. while hasattr(type_, '__supertype__'):
  371. type_ = type_.__supertype__
  372. return type_
  373. def _check_classvar(v: Optional[Type[Any]]) -> bool:
  374. if v is None:
  375. return False
  376. return v.__class__ == ClassVar.__class__ and getattr(v, '_name', None) == 'ClassVar'
  377. def _check_finalvar(v: Optional[Type[Any]]) -> bool:
  378. """
  379. Check if a given type is a `typing.Final` type.
  380. """
  381. if v is None:
  382. return False
  383. return v.__class__ == Final.__class__ and (sys.version_info < (3, 8) or getattr(v, '_name', None) == 'Final')
  384. def is_classvar(ann_type: Type[Any]) -> bool:
  385. if _check_classvar(ann_type) or _check_classvar(get_origin(ann_type)):
  386. return True
  387. # this is an ugly workaround for class vars that contain forward references and are therefore themselves
  388. # forward references, see #3679
  389. if ann_type.__class__ == ForwardRef and ann_type.__forward_arg__.startswith('ClassVar['):
  390. return True
  391. return False
  392. def is_finalvar(ann_type: Type[Any]) -> bool:
  393. return _check_finalvar(ann_type) or _check_finalvar(get_origin(ann_type))
  394. def update_field_forward_refs(field: 'ModelField', globalns: Any, localns: Any) -> None:
  395. """
  396. Try to update ForwardRefs on fields based on this ModelField, globalns and localns.
  397. """
  398. prepare = False
  399. if field.type_.__class__ == ForwardRef:
  400. prepare = True
  401. field.type_ = evaluate_forwardref(field.type_, globalns, localns or None)
  402. if field.outer_type_.__class__ == ForwardRef:
  403. prepare = True
  404. field.outer_type_ = evaluate_forwardref(field.outer_type_, globalns, localns or None)
  405. if prepare:
  406. field.prepare()
  407. if field.sub_fields:
  408. for sub_f in field.sub_fields:
  409. update_field_forward_refs(sub_f, globalns=globalns, localns=localns)
  410. if field.discriminator_key is not None:
  411. field.prepare_discriminated_union_sub_fields()
  412. def update_model_forward_refs(
  413. model: Type[Any],
  414. fields: Iterable['ModelField'],
  415. json_encoders: Dict[Union[Type[Any], str, ForwardRef], AnyCallable],
  416. localns: 'DictStrAny',
  417. exc_to_suppress: Tuple[Type[BaseException], ...] = (),
  418. ) -> None:
  419. """
  420. Try to update model fields ForwardRefs based on model and localns.
  421. """
  422. if model.__module__ in sys.modules:
  423. globalns = sys.modules[model.__module__].__dict__.copy()
  424. else:
  425. globalns = {}
  426. globalns.setdefault(model.__name__, model)
  427. for f in fields:
  428. try:
  429. update_field_forward_refs(f, globalns=globalns, localns=localns)
  430. except exc_to_suppress:
  431. pass
  432. for key in set(json_encoders.keys()):
  433. if isinstance(key, str):
  434. fr: ForwardRef = ForwardRef(key)
  435. elif isinstance(key, ForwardRef):
  436. fr = key
  437. else:
  438. continue
  439. try:
  440. new_key = evaluate_forwardref(fr, globalns, localns or None)
  441. except exc_to_suppress: # pragma: no cover
  442. continue
  443. json_encoders[new_key] = json_encoders.pop(key)
  444. def get_class(type_: Type[Any]) -> Union[None, bool, Type[Any]]:
  445. """
  446. Tries to get the class of a Type[T] annotation. Returns True if Type is used
  447. without brackets. Otherwise returns None.
  448. """
  449. if type_ is type:
  450. return True
  451. if get_origin(type_) is None:
  452. return None
  453. args = get_args(type_)
  454. if not args or not isinstance(args[0], type):
  455. return True
  456. else:
  457. return args[0]
  458. def get_sub_types(tp: Any) -> List[Any]:
  459. """
  460. Return all the types that are allowed by type `tp`
  461. `tp` can be a `Union` of allowed types or an `Annotated` type
  462. """
  463. origin = get_origin(tp)
  464. if origin is Annotated:
  465. return get_sub_types(get_args(tp)[0])
  466. elif is_union(origin):
  467. return [x for t in get_args(tp) for x in get_sub_types(t)]
  468. else:
  469. return [tp]