markers.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576
  1. # This file is dual licensed under the terms of the Apache License, Version
  2. # 2.0, and the BSD License. See the LICENSE file in the root of this repository
  3. # for complete details.
  4. from __future__ import annotations
  5. import functools
  6. import operator
  7. import os
  8. import platform
  9. import sys
  10. from collections.abc import Set as AbstractSet
  11. from typing import TYPE_CHECKING, Callable, Literal, TypedDict, Union, cast
  12. from ._parser import MarkerAtom, MarkerList, Op, Value, Variable
  13. from ._parser import parse_marker as _parse_marker
  14. from ._tokenizer import ParserSyntaxError
  15. from .specifiers import InvalidSpecifier, Specifier
  16. from .utils import canonicalize_name
  17. if TYPE_CHECKING:
  18. from collections.abc import Mapping
  19. __all__ = [
  20. "Environment",
  21. "EvaluateContext",
  22. "InvalidMarker",
  23. "Marker",
  24. "UndefinedComparison",
  25. "UndefinedEnvironmentName",
  26. "default_environment",
  27. ]
  28. def __dir__() -> list[str]:
  29. return __all__
  30. Operator = Callable[[str, Union[str, AbstractSet[str]]], bool]
  31. EvaluateContext = Literal["metadata", "lock_file", "requirement"]
  32. """A ``typing.Literal`` enumerating valid marker evaluation contexts.
  33. Valid values for the ``context`` passed to :meth:`Marker.evaluate` are:
  34. * ``"metadata"`` (for core metadata; default)
  35. * ``"lock_file"`` (for lock files)
  36. * ``"requirement"`` (i.e. all other situations)
  37. .. versionadded:: 25.0
  38. """
  39. MARKERS_ALLOWING_SET = {"extras", "dependency_groups"}
  40. MARKERS_REQUIRING_VERSION = {
  41. "implementation_version",
  42. "platform_release",
  43. "python_full_version",
  44. "python_version",
  45. }
  46. class InvalidMarker(ValueError):
  47. """Raised when attempting to create a :class:`Marker` from invalid input.
  48. This error indicates that the given marker string does not conform to the
  49. :ref:`specification of dependency specifiers <pypug:dependency-specifiers>`.
  50. """
  51. class UndefinedComparison(ValueError):
  52. """Raised when evaluating an unsupported marker comparison.
  53. This can happen when marker values are compared as versions but do not
  54. conform to the :ref:`specification of version specifiers
  55. <pypug:version-specifiers>`.
  56. """
  57. class UndefinedEnvironmentName(KeyError):
  58. """Raised when evaluating a marker that references a missing environment key.
  59. Subclasses :class:`KeyError` so that code catching the bare ``KeyError`` that
  60. a missing environment lookup historically produced keeps working.
  61. .. versionchanged:: 26.3
  62. Now subclasses :class:`KeyError` (was :class:`ValueError`) and is raised by
  63. :meth:`Marker.evaluate` for missing environment keys, where a bare
  64. ``KeyError`` was raised before.
  65. """
  66. class Environment(TypedDict):
  67. """
  68. A dictionary that represents a Python environment as captured by
  69. :func:`default_environment`. All fields are required.
  70. """
  71. implementation_name: str
  72. """The implementation's identifier, e.g. ``'cpython'``."""
  73. implementation_version: str
  74. """
  75. The implementation's version, e.g. ``'3.13.0a2'`` for CPython 3.13.0a2, or
  76. ``'7.3.13'`` for PyPy3.10 v7.3.13.
  77. """
  78. os_name: str
  79. """
  80. The value of :py:data:`os.name`. The name of the operating system dependent module
  81. imported, e.g. ``'posix'``.
  82. """
  83. platform_machine: str
  84. """
  85. Returns the machine type, e.g. ``'i386'``.
  86. An empty string if the value cannot be determined.
  87. """
  88. platform_release: str
  89. """
  90. The system's release, e.g. ``'2.2.0'`` or ``'NT'``.
  91. An empty string if the value cannot be determined.
  92. """
  93. platform_system: str
  94. """
  95. The system/OS name, e.g. ``'Linux'``, ``'Windows'`` or ``'Java'``.
  96. An empty string if the value cannot be determined.
  97. """
  98. platform_version: str
  99. """
  100. The system's release version, e.g. ``'#3 on degas'``.
  101. An empty string if the value cannot be determined.
  102. """
  103. python_full_version: str
  104. """
  105. The Python version as string ``'major.minor.patchlevel'``.
  106. Note that unlike the Python :py:data:`sys.version`, this value will always include
  107. the patchlevel (it defaults to 0).
  108. """
  109. platform_python_implementation: str
  110. """
  111. A string identifying the Python implementation, e.g. ``'CPython'``.
  112. """
  113. python_version: str
  114. """The Python version as string ``'major.minor'``."""
  115. sys_platform: str
  116. """
  117. This string contains a platform identifier that can be used to append
  118. platform-specific components to :py:data:`sys.path`, for instance.
  119. For Unix systems, except on Linux and AIX, this is the lowercased OS name as
  120. returned by ``uname -s`` with the first part of the version as returned by
  121. ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, at the time when Python
  122. was built.
  123. """
  124. def _normalize_extras(
  125. result: MarkerList | MarkerAtom | str,
  126. ) -> MarkerList | MarkerAtom | str:
  127. if isinstance(result, list):
  128. return [_normalize_extras(r) for r in result]
  129. if not isinstance(result, tuple):
  130. return result
  131. lhs, op, rhs = result
  132. if isinstance(lhs, Variable) and lhs.value == "extra" and isinstance(rhs, Value):
  133. normalized_extra = canonicalize_name(rhs.value)
  134. rhs = Value(normalized_extra)
  135. elif isinstance(rhs, Variable) and rhs.value == "extra" and isinstance(lhs, Value):
  136. normalized_extra = canonicalize_name(lhs.value)
  137. lhs = Value(normalized_extra)
  138. elif (
  139. isinstance(rhs, Variable)
  140. and rhs.value in MARKERS_ALLOWING_SET
  141. and isinstance(lhs, Value)
  142. ):
  143. # PEP 685 (extras) / PEP 735 (dependency_groups): the set-valued membership
  144. # literal must also be normalized. evaluate() already canonicalizes both
  145. # operands for these keys (see _normalize), so normalizing the literal at
  146. # parse time keeps __str__/__eq__/__hash__ consistent with evaluate() -- e.g.
  147. # Marker('"Foo" in extras') and Marker('"foo" in extras') must compare and
  148. # hash equal (the membership variable is always the right-hand operand).
  149. lhs = Value(canonicalize_name(lhs.value))
  150. return lhs, op, rhs
  151. def _normalize_extra_values(results: MarkerList) -> MarkerList:
  152. """
  153. Normalize extra values.
  154. """
  155. return [_normalize_extras(r) for r in results]
  156. def _format_marker(
  157. marker: list[str] | MarkerAtom | str, first: bool | None = True
  158. ) -> str:
  159. assert isinstance(marker, (list, tuple, str))
  160. # Unwrap a redundant [[...]] wrapper, but keep the nesting context so a
  161. # nested group keeps the parentheses its and/or precedence needs.
  162. if (
  163. isinstance(marker, list)
  164. and len(marker) == 1
  165. and isinstance(marker[0], (list, tuple))
  166. ):
  167. return _format_marker(marker[0], first=first)
  168. if isinstance(marker, list):
  169. inner = (_format_marker(m, first=False) for m in marker)
  170. if first:
  171. return " ".join(inner)
  172. else:
  173. return "(" + " ".join(inner) + ")"
  174. elif isinstance(marker, tuple):
  175. return " ".join([m.serialize() for m in marker])
  176. else:
  177. return marker
  178. _operators: dict[str, Operator] = {
  179. "in": lambda lhs, rhs: lhs in rhs,
  180. "not in": lambda lhs, rhs: lhs not in rhs,
  181. "<": lambda _lhs, _rhs: False,
  182. "<=": operator.eq,
  183. "==": operator.eq,
  184. "!=": operator.ne,
  185. ">=": operator.eq,
  186. ">": lambda _lhs, _rhs: False,
  187. }
  188. def _eval_op(lhs: str, op: Op, rhs: str | AbstractSet[str], *, key: str) -> bool:
  189. op_str = op.serialize()
  190. if key in MARKERS_REQUIRING_VERSION:
  191. try:
  192. spec = Specifier(f"{op_str}{rhs}")
  193. except InvalidSpecifier:
  194. pass
  195. else:
  196. return spec.contains(lhs, prereleases=True)
  197. oper: Operator | None = _operators.get(op_str)
  198. if oper is None:
  199. raise UndefinedComparison(f"Undefined {op!r} on {lhs!r} and {rhs!r}.")
  200. return oper(lhs, rhs)
  201. def _normalize(
  202. lhs: str, rhs: str | AbstractSet[str], key: str
  203. ) -> tuple[str, str | AbstractSet[str]]:
  204. # PEP 685 - Comparison of extra names for optional distribution dependencies
  205. # https://peps.python.org/pep-0685/
  206. # > When comparing extra names, tools MUST normalize the names being
  207. # > compared using the semantics outlined in PEP 503 for names
  208. if key == "extra":
  209. assert isinstance(rhs, str), "extra value must be a string"
  210. # Both sides are normalized at this point already
  211. return (lhs, rhs)
  212. if key in MARKERS_ALLOWING_SET:
  213. if isinstance(rhs, str): # pragma: no cover
  214. return (canonicalize_name(lhs), canonicalize_name(rhs))
  215. else:
  216. return (canonicalize_name(lhs), {canonicalize_name(v) for v in rhs})
  217. # other environment markers don't have such standards
  218. return lhs, rhs
  219. def _lookup_environment(
  220. environment: dict[str, str | AbstractSet[str]], key: str
  221. ) -> str | AbstractSet[str]:
  222. try:
  223. return environment[key]
  224. except KeyError:
  225. raise UndefinedEnvironmentName(key) from None
  226. def _evaluate_markers(
  227. markers: MarkerList, environment: dict[str, str | AbstractSet[str]]
  228. ) -> bool:
  229. groups: list[list[bool]] = [[]]
  230. for marker in markers:
  231. if isinstance(marker, list):
  232. groups[-1].append(_evaluate_markers(marker, environment))
  233. elif isinstance(marker, tuple):
  234. lhs, op, rhs = marker
  235. if isinstance(lhs, Variable):
  236. environment_key = lhs.value
  237. lhs_value = _lookup_environment(environment, environment_key)
  238. rhs_value = rhs.value
  239. else:
  240. lhs_value = lhs.value
  241. environment_key = rhs.value
  242. rhs_value = _lookup_environment(environment, environment_key)
  243. if not isinstance(lhs_value, str):
  244. raise UndefinedComparison(
  245. f"Set-valued marker {environment_key!r} can only be used "
  246. f'with the membership form (e.g. "<name>" in '
  247. f"{environment_key}); it cannot appear on the left-hand "
  248. f"side of {op.serialize()!r}."
  249. )
  250. lhs_value, rhs_value = _normalize(lhs_value, rhs_value, key=environment_key)
  251. groups[-1].append(_eval_op(lhs_value, op, rhs_value, key=environment_key))
  252. elif marker == "or":
  253. groups.append([])
  254. elif marker == "and":
  255. pass
  256. else: # pragma: nocover
  257. raise TypeError(f"Unexpected marker {marker!r}")
  258. return any(all(item) for item in groups)
  259. def _format_full_version(info: sys._version_info) -> str:
  260. version = f"{info.major}.{info.minor}.{info.micro}"
  261. kind = info.releaselevel
  262. if kind != "final":
  263. version += kind[0] + str(info.serial)
  264. return version
  265. @functools.cache
  266. def _cached_default_environment() -> Environment:
  267. """Build the default marker environment for the current Python process.
  268. The values are derived from process-constant data (the running interpreter
  269. and the host platform), so this is cached and built only once. The result is
  270. shared between callers and must never be mutated; :func:`default_environment`
  271. returns a fresh copy.
  272. """
  273. iver = _format_full_version(sys.implementation.version)
  274. implementation_name = sys.implementation.name
  275. return {
  276. "implementation_name": implementation_name,
  277. "implementation_version": iver,
  278. "os_name": os.name,
  279. "platform_machine": platform.machine(),
  280. "platform_release": platform.release(),
  281. "platform_system": platform.system(),
  282. "platform_version": platform.version(),
  283. "python_full_version": platform.python_version(),
  284. "platform_python_implementation": platform.python_implementation(),
  285. "python_version": ".".join(platform.python_version_tuple()[:2]),
  286. "sys_platform": sys.platform,
  287. }
  288. def default_environment() -> Environment:
  289. """Return the default marker environment for the current Python process.
  290. This is the base environment used by :meth:`Marker.evaluate`. A fresh copy
  291. is returned on every call so callers may freely mutate the result; a shallow
  292. copy suffices because all values are immutable strings.
  293. .. versionchanged:: 26.3
  294. The environment is computed once per process and cached, since it is
  295. derived from process-constant data. Patching ``platform``/``sys``/``os``
  296. after the first call has no effect; pass an explicit ``environment`` to
  297. :meth:`Marker.evaluate` to evaluate against different values.
  298. """
  299. return cast("Environment", dict(_cached_default_environment()))
  300. class Marker:
  301. """Represents a parsed dependency marker expression.
  302. Marker expressions are parsed according to the
  303. :ref:`specification of dependency specifiers <pypug:dependency-specifiers>`.
  304. :param marker: The string representation of a marker expression.
  305. :raises InvalidMarker: If ``marker`` cannot be parsed.
  306. Instances are safe to serialize with :mod:`pickle`. They use a stable
  307. format so the same pickle can be loaded in future packaging releases.
  308. .. versionchanged:: 26.2
  309. Added a stable pickle format. Pickles created with packaging 26.2+ can
  310. be unpickled with future releases. Backward compatibility with pickles
  311. from packaging < 26.2 is supported but may be removed in a future
  312. release.
  313. """
  314. __slots__ = ("_markers",)
  315. def __init__(self, marker: str) -> None:
  316. # Note: We create a Marker object without calling this constructor in
  317. # packaging.requirements.Requirement. If any additional logic is
  318. # added here, make sure to mirror/adapt Requirement.
  319. # If this fails and throws an error, the repr still expects _markers to
  320. # be defined.
  321. self._markers: MarkerList = []
  322. try:
  323. self._markers = _normalize_extra_values(_parse_marker(marker))
  324. # The attribute `_markers` can be described in terms of a recursive type:
  325. # MarkerList = List[Union[Tuple[Node, ...], str, MarkerList]]
  326. #
  327. # For example, the following expression:
  328. # python_version > "3.6" or (python_version == "3.6" and os_name == "unix")
  329. #
  330. # is parsed into:
  331. # [
  332. # (<Variable('python_version')>, <Op('>')>, <Value('3.6')>),
  333. # 'and',
  334. # [
  335. # (<Variable('python_version')>, <Op('==')>, <Value('3.6')>),
  336. # 'or',
  337. # (<Variable('os_name')>, <Op('==')>, <Value('unix')>)
  338. # ]
  339. # ]
  340. except ParserSyntaxError as e:
  341. raise InvalidMarker(str(e)) from e
  342. @classmethod
  343. def _from_markers(cls, markers: MarkerList) -> Marker:
  344. """Create a Marker instance from a pre-parsed marker tree.
  345. This avoids re-parsing serialised marker strings when combining markers.
  346. """
  347. new = cls.__new__(cls)
  348. new._markers = markers
  349. return new
  350. def __str__(self) -> str:
  351. return _format_marker(self._markers)
  352. def __repr__(self) -> str:
  353. return f"<{self.__class__.__name__}({str(self)!r})>"
  354. def __hash__(self) -> int:
  355. return hash(str(self))
  356. def __eq__(self, other: object) -> bool:
  357. if not isinstance(other, Marker):
  358. return NotImplemented
  359. return str(self) == str(other)
  360. def __getstate__(self) -> str:
  361. # Return the marker expression string for compactness and stability.
  362. # Internal Node objects are excluded; the string is re-parsed on load.
  363. return str(self)
  364. def __setstate__(self, state: object) -> None:
  365. if isinstance(state, str):
  366. # New format (26.2+): just the marker expression string.
  367. try:
  368. self._markers = _normalize_extra_values(_parse_marker(state))
  369. except ParserSyntaxError as exc:
  370. raise TypeError(f"Cannot restore Marker from {state!r}") from exc
  371. return
  372. if isinstance(state, dict) and "_markers" in state:
  373. # Old format (packaging <= 26.1, no __slots__): plain __dict__.
  374. markers = state["_markers"]
  375. if isinstance(markers, list):
  376. self._markers = markers
  377. return
  378. if isinstance(state, tuple) and len(state) == 2:
  379. # Old format (packaging <= 26.1, __slots__): (None, {slot: value}).
  380. _, slot_dict = state
  381. if isinstance(slot_dict, dict) and "_markers" in slot_dict:
  382. markers = slot_dict["_markers"]
  383. if isinstance(markers, list):
  384. self._markers = markers
  385. return
  386. raise TypeError(f"Cannot restore Marker from {state!r}")
  387. def __and__(self, other: Marker) -> Marker:
  388. """Combine this marker with another using ``and``.
  389. .. versionadded:: 26.1
  390. """
  391. if not isinstance(other, Marker):
  392. return NotImplemented
  393. return self._from_markers([self._markers, "and", other._markers])
  394. def __or__(self, other: Marker) -> Marker:
  395. """Combine this marker with another using ``or``.
  396. .. versionadded:: 26.1
  397. """
  398. if not isinstance(other, Marker):
  399. return NotImplemented
  400. return self._from_markers([self._markers, "or", other._markers])
  401. def evaluate(
  402. self,
  403. environment: Mapping[str, str | AbstractSet[str]] | None = None,
  404. context: EvaluateContext = "metadata",
  405. ) -> bool:
  406. """Evaluate a marker.
  407. Return the boolean from evaluating this marker against the environment.
  408. The environment is determined from the current Python process unless
  409. passed in explicitly.
  410. :param environment: Mapping containing keys and values to override the
  411. detected environment.
  412. :param EvaluateContext context: The context in which the marker is
  413. evaluated, which influences what marker names are considered valid.
  414. Accepted values are ``"metadata"`` (for core metadata; default),
  415. ``"lock_file"``, and ``"requirement"`` (i.e. all other situations).
  416. :raises UndefinedComparison: If the marker uses a comparison on values
  417. that are not valid versions per the :ref:`specification of version
  418. specifiers <pypug:version-specifiers>`.
  419. :raises UndefinedEnvironmentName: If the marker references a value that
  420. is missing from the evaluation environment.
  421. :returns: ``True`` if the marker matches, otherwise ``False``.
  422. .. versionchanged:: 25.0
  423. Added the ``context`` parameter, which influences which marker names
  424. are considered valid.
  425. """
  426. current_environment = cast(
  427. "dict[str, str | AbstractSet[str]]", default_environment()
  428. )
  429. if context == "lock_file":
  430. current_environment |= {
  431. "extras": frozenset(),
  432. "dependency_groups": frozenset(),
  433. }
  434. elif context == "metadata":
  435. current_environment["extra"] = ""
  436. if environment is not None:
  437. current_environment |= environment
  438. if "extra" in current_environment:
  439. # The API used to allow setting extra to None. We need to handle
  440. # this case for backwards compatibility. Also skip running
  441. # normalize name if extra is empty.
  442. extra = cast("str | None", current_environment["extra"])
  443. current_environment["extra"] = canonicalize_name(extra) if extra else ""
  444. return _evaluate_markers(
  445. self._markers, _repair_python_full_version(current_environment)
  446. )
  447. def _pep440_python_full_version(python_full_version: str) -> str:
  448. """
  449. Work around platform.python_version() returning something that is not PEP 440
  450. compliant for non-tagged Python builds.
  451. """
  452. if python_full_version.endswith("+"):
  453. return f"{python_full_version}local"
  454. return python_full_version
  455. def _repair_python_full_version(
  456. env: dict[str, str | AbstractSet[str]],
  457. ) -> dict[str, str | AbstractSet[str]]:
  458. """
  459. Work around platform.python_version() returning something that is not PEP 440
  460. compliant for non-tagged Python builds.
  461. """
  462. python_full_version = cast("str", env["python_full_version"])
  463. env["python_full_version"] = _pep440_python_full_version(python_full_version)
  464. return env