metadata.py 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054
  1. from __future__ import annotations
  2. import email.header
  3. import email.message
  4. import email.parser
  5. import email.policy
  6. import keyword
  7. import pathlib
  8. import re
  9. import typing
  10. from typing import (
  11. Any,
  12. Callable,
  13. Generic,
  14. Literal,
  15. TypedDict,
  16. cast,
  17. )
  18. from . import licenses, requirements, specifiers, utils
  19. from . import version as version_module
  20. from .errors import ExceptionGroup, _ErrorCollector
  21. if typing.TYPE_CHECKING:
  22. from .licenses import NormalizedLicenseExpression
  23. from .version import Version
  24. T = typing.TypeVar("T")
  25. __all__ = [
  26. "ExceptionGroup", # Keep this for a bit (makes mypy happy w/ 26.0 compat)
  27. "InvalidMetadata",
  28. "Metadata",
  29. "RFC822Message",
  30. "RFC822Policy",
  31. "RawMetadata",
  32. "parse_email",
  33. ]
  34. def __dir__() -> list[str]:
  35. return __all__
  36. class InvalidMetadata(ValueError):
  37. """A metadata field contains invalid data.
  38. .. versionadded:: 23.2
  39. """
  40. field: str
  41. """The name of the field that contains invalid data."""
  42. def __init__(self, field: str, message: str) -> None:
  43. self.field = field
  44. super().__init__(message)
  45. # Support pickling; ``args`` does not match ``__init__``'s signature.
  46. def __reduce__(self) -> tuple[type[InvalidMetadata], tuple[str, str]]:
  47. return (self.__class__, (self.field, self.args[0]))
  48. # The RawMetadata class attempts to make as few assumptions about the underlying
  49. # serialization formats as possible. The idea is that as long as a serialization
  50. # formats offer some very basic primitives in *some* way then we can support
  51. # serializing to and from that format.
  52. class RawMetadata(TypedDict, total=False):
  53. """A dictionary of raw core metadata.
  54. Each field in core metadata maps to a key of this dictionary (when data is
  55. provided). The key is lower-case and underscores are used instead of dashes
  56. compared to the equivalent core metadata field. Any core metadata field that
  57. can be specified multiple times or can hold multiple values in a single
  58. field have a key with a plural name. See :class:`Metadata` whose attributes
  59. match the keys of this dictionary.
  60. Core metadata fields that can be specified multiple times are stored as a
  61. list or dict depending on which is appropriate for the field. Any fields
  62. which hold multiple values in a single field are stored as a list. All fields
  63. are considered optional.
  64. """
  65. # Metadata 1.0 - PEP 241
  66. metadata_version: str
  67. name: str
  68. version: str
  69. platforms: list[str]
  70. summary: str
  71. description: str
  72. keywords: list[str]
  73. home_page: str
  74. author: str
  75. author_email: str
  76. license: str
  77. # Metadata 1.1 - PEP 314
  78. supported_platforms: list[str]
  79. download_url: str
  80. classifiers: list[str]
  81. requires: list[str]
  82. provides: list[str]
  83. obsoletes: list[str]
  84. # Metadata 1.2 - PEP 345
  85. maintainer: str
  86. maintainer_email: str
  87. requires_dist: list[str]
  88. provides_dist: list[str]
  89. obsoletes_dist: list[str]
  90. requires_python: str
  91. requires_external: list[str]
  92. project_urls: dict[str, str]
  93. # Metadata 2.0
  94. # PEP 426 attempted to completely revamp the metadata format
  95. # but got stuck without ever being able to build consensus on
  96. # it and ultimately ended up withdrawn.
  97. #
  98. # However, a number of tools had started emitting METADATA with
  99. # `2.0` Metadata-Version, so for historical reasons, this version
  100. # was skipped.
  101. # Metadata 2.1 - PEP 566
  102. description_content_type: str
  103. provides_extra: list[str]
  104. # Metadata 2.2 - PEP 643
  105. dynamic: list[str]
  106. # Metadata 2.3 - PEP 685
  107. # No new fields were added in PEP 685, just some edge case were
  108. # tightened up to provide better interoperability.
  109. # Metadata 2.4 - PEP 639
  110. license_expression: str
  111. """.. versionadded:: 24.2"""
  112. license_files: list[str]
  113. """.. versionadded:: 24.2"""
  114. # Metadata 2.5 - PEP 794
  115. import_names: list[str]
  116. """.. versionadded:: 26.0"""
  117. import_namespaces: list[str]
  118. """.. versionadded:: 26.0"""
  119. # Metadata 2.6 - PEP 808 (no new fields, behavior change for Dynamic)
  120. # 'keywords' is special as it's a string in the core metadata spec, but we
  121. # represent it as a list.
  122. _STRING_FIELDS = {
  123. "author",
  124. "author_email",
  125. "description",
  126. "description_content_type",
  127. "download_url",
  128. "home_page",
  129. "license",
  130. "license_expression",
  131. "maintainer",
  132. "maintainer_email",
  133. "metadata_version",
  134. "name",
  135. "requires_python",
  136. "summary",
  137. "version",
  138. }
  139. _LIST_FIELDS = {
  140. "classifiers",
  141. "dynamic",
  142. "license_files",
  143. "obsoletes",
  144. "obsoletes_dist",
  145. "platforms",
  146. "provides",
  147. "provides_dist",
  148. "provides_extra",
  149. "requires",
  150. "requires_dist",
  151. "requires_external",
  152. "supported_platforms",
  153. "import_names",
  154. "import_namespaces",
  155. }
  156. _DICT_FIELDS = {
  157. "project_urls",
  158. }
  159. def _parse_keywords(data: str) -> list[str]:
  160. """Split a string of comma-separated keywords into a list of keywords."""
  161. return [k.strip() for k in data.split(",")]
  162. def _parse_project_urls(data: list[str]) -> dict[str, str]:
  163. """Parse a list of label/URL string pairings separated by a comma."""
  164. urls = {}
  165. for pair in data:
  166. # Our logic is slightly tricky here as we want to try and do
  167. # *something* reasonable with malformed data.
  168. #
  169. # The main thing that we have to worry about, is data that does
  170. # not have a ',' at all to split the label from the Value. There
  171. # isn't a singular right answer here, and we will fail validation
  172. # later on (if the caller is validating) so it doesn't *really*
  173. # matter, but since the missing value has to be an empty str
  174. # and our return value is dict[str, str], if we let the key
  175. # be the missing value, then they'd have multiple '' values that
  176. # overwrite each other in a accumulating dict.
  177. #
  178. # The other potential issue is that it's possible to have the
  179. # same label multiple times in the metadata, with no solid "right"
  180. # answer with what to do in that case. As such, we'll do the only
  181. # thing we can, which is treat the field as unparsable and add it
  182. # to our list of unparsed fields.
  183. #
  184. # TODO: The spec doesn't say anything about if the keys should be
  185. # considered case sensitive or not... logically they should
  186. # be case-preserving and case-insensitive, but doing that
  187. # would open up more cases where we might have duplicate
  188. # entries.
  189. label, _, url = (s.strip() for s in pair.partition(","))
  190. if label in urls:
  191. # The label already exists in our set of urls, so this field
  192. # is unparsable, and we can just add the whole thing to our
  193. # unparsable data and stop processing it.
  194. raise KeyError("duplicate labels in project urls")
  195. urls[label] = url
  196. return urls
  197. def _get_payload(msg: email.message.Message, source: bytes | str) -> str:
  198. """Get the body of the message."""
  199. # If our source is a str, then our caller has managed encodings for us,
  200. # and we don't need to deal with it.
  201. if isinstance(source, str):
  202. payload = msg.get_payload()
  203. # A multipart payload makes get_payload() return a list of messages
  204. # rather than a str; route it to ``unparsed``.
  205. if not isinstance(payload, str):
  206. raise ValueError("payload is not a string") # noqa: TRY004
  207. return payload
  208. # If our source is a bytes, then we're managing the encoding and we need
  209. # to deal with it.
  210. else:
  211. bpayload = msg.get_payload(decode=True)
  212. # A multipart payload makes get_payload(decode=True) return None;
  213. # route it to ``unparsed``.
  214. if not isinstance(bpayload, bytes):
  215. raise ValueError("payload in an invalid encoding") # noqa: TRY004
  216. try:
  217. return bpayload.decode("utf8", "strict")
  218. except UnicodeDecodeError as exc:
  219. raise ValueError("payload in an invalid encoding") from exc
  220. # The various parse_FORMAT functions here are intended to be as lenient as
  221. # possible in their parsing, while still returning a correctly typed
  222. # RawMetadata.
  223. #
  224. # To aid in this, we also generally want to do as little touching of the
  225. # data as possible, except where there are possibly some historic holdovers
  226. # that make valid data awkward to work with.
  227. #
  228. # While this is a lower level, intermediate format than our ``Metadata``
  229. # class, some light touch ups can make a massive difference in usability.
  230. # Map METADATA fields to RawMetadata.
  231. _EMAIL_TO_RAW_MAPPING = {
  232. "author": "author",
  233. "author-email": "author_email",
  234. "classifier": "classifiers",
  235. "description": "description",
  236. "description-content-type": "description_content_type",
  237. "download-url": "download_url",
  238. "dynamic": "dynamic",
  239. "home-page": "home_page",
  240. "import-name": "import_names",
  241. "import-namespace": "import_namespaces",
  242. "keywords": "keywords",
  243. "license": "license",
  244. "license-expression": "license_expression",
  245. "license-file": "license_files",
  246. "maintainer": "maintainer",
  247. "maintainer-email": "maintainer_email",
  248. "metadata-version": "metadata_version",
  249. "name": "name",
  250. "obsoletes": "obsoletes",
  251. "obsoletes-dist": "obsoletes_dist",
  252. "platform": "platforms",
  253. "project-url": "project_urls",
  254. "provides": "provides",
  255. "provides-dist": "provides_dist",
  256. "provides-extra": "provides_extra",
  257. "requires": "requires",
  258. "requires-dist": "requires_dist",
  259. "requires-external": "requires_external",
  260. "requires-python": "requires_python",
  261. "summary": "summary",
  262. "supported-platform": "supported_platforms",
  263. "version": "version",
  264. }
  265. _RAW_TO_EMAIL_MAPPING = {raw: email for email, raw in _EMAIL_TO_RAW_MAPPING.items()}
  266. # A bare "\r" makes the email generator raise ``HeaderWriteError``, and on
  267. # CPython releases without the CVE-2024-6923 fix any ``str.splitlines``
  268. # boundary ends the header line, so fold all of them, not just "\n".
  269. _LINE_BOUNDARY_RE = re.compile(r"\r\n|[\n\r\v\f\x1c\x1d\x1e\x85\u2028\u2029]")
  270. # This class is for writing RFC822 messages
  271. class RFC822Policy(email.policy.EmailPolicy):
  272. """
  273. This is :class:`email.policy.EmailPolicy`, but with a simple ``header_store_parse``
  274. implementation that handles multi-line values, and some nice defaults.
  275. .. versionadded:: 26.0
  276. """
  277. utf8 = True
  278. mangle_from_ = False
  279. max_line_length = 0
  280. def header_store_parse(self, name: str, value: str) -> tuple[str, str]:
  281. size = len(name) + 2
  282. value = _LINE_BOUNDARY_RE.sub("\n" + " " * size, value)
  283. return (name, value)
  284. # This class is for writing RFC822 messages
  285. class RFC822Message(email.message.EmailMessage):
  286. """
  287. This is :class:`email.message.EmailMessage` with two small changes: it defaults to
  288. our `RFC822Policy`, and it correctly writes unicode when being called
  289. with `bytes()`.
  290. .. versionadded:: 26.0
  291. """
  292. def __init__(self) -> None:
  293. super().__init__(policy=RFC822Policy())
  294. def as_bytes(
  295. self, unixfrom: bool = False, policy: email.policy.Policy | None = None
  296. ) -> bytes:
  297. """
  298. Return the bytes representation of the message.
  299. This handles unicode encoding.
  300. """
  301. return self.as_string(unixfrom, policy=policy).encode("utf-8")
  302. def parse_email(data: bytes | str) -> tuple[RawMetadata, dict[str, list[str]]]:
  303. """Parse a distribution's metadata stored as email headers (e.g. from ``METADATA``).
  304. This function returns a two-item tuple of dicts. The first dict is of
  305. recognized fields from the core metadata specification. Fields that can be
  306. parsed and translated into Python's built-in types are converted
  307. appropriately. All other fields are left as-is. Fields that are allowed to
  308. appear multiple times are stored as lists.
  309. The second dict contains all other fields from the metadata. This includes
  310. any unrecognized fields. It also includes any fields which are expected to
  311. be parsed into a built-in type but were not formatted appropriately. Finally,
  312. any fields that are expected to appear only once but are repeated are
  313. included in this dict.
  314. """
  315. raw: dict[str, str | list[str] | dict[str, str]] = {}
  316. unparsed: dict[str, list[str]] = {}
  317. if isinstance(data, str):
  318. parsed = email.parser.Parser(policy=email.policy.compat32).parsestr(data)
  319. else:
  320. parsed = email.parser.BytesParser(policy=email.policy.compat32).parsebytes(data)
  321. # We have to wrap parsed.keys() in a set, because in the case of multiple
  322. # values for a key (a list), the key will appear multiple times in the
  323. # list of keys, but we're avoiding that by using get_all().
  324. for name_with_case in frozenset(parsed.keys()):
  325. # Header names in RFC are case insensitive, so we'll normalize to all
  326. # lower case to make comparisons easier.
  327. name = name_with_case.lower()
  328. # We use get_all() here, even for fields that aren't multiple use,
  329. # because otherwise someone could have e.g. two Name fields, and we
  330. # would just silently ignore it rather than doing something about it.
  331. headers = parsed.get_all(name) or []
  332. # The way the email module works when parsing bytes is that it
  333. # unconditionally decodes the bytes as ascii using the surrogateescape
  334. # handler. When you pull that data back out (such as with get_all() ),
  335. # it looks to see if the str has any surrogate escapes, and if it does
  336. # it wraps it in a Header object instead of returning the string.
  337. #
  338. # As such, we'll look for those Header objects, and fix up the encoding.
  339. value = []
  340. # Flag if we have run into any issues processing the headers, thus
  341. # signalling that the data belongs in 'unparsed'.
  342. valid_encoding = True
  343. for h in headers:
  344. # It's unclear if this can return more types than just a Header or
  345. # a str, so we'll just assert here to make sure.
  346. assert isinstance(h, (email.header.Header, str))
  347. # If it's a header object, we need to do our little dance to get
  348. # the real data out of it. In cases where there is invalid data
  349. # we're going to end up with mojibake, but there's no obvious, good
  350. # way around that without reimplementing parts of the Header object
  351. # ourselves.
  352. #
  353. # That should be fine since, if mojibacked happens, this key is
  354. # going into the unparsed dict anyways.
  355. if isinstance(h, email.header.Header):
  356. # The Header object stores it's data as chunks, and each chunk
  357. # can be independently encoded, so we'll need to check each
  358. # of them.
  359. chunks: list[tuple[bytes, str | None]] = []
  360. for binary, _encoding in email.header.decode_header(h):
  361. try:
  362. binary.decode("utf8", "strict")
  363. except UnicodeDecodeError:
  364. # Enable mojibake.
  365. encoding = "latin1"
  366. valid_encoding = False
  367. else:
  368. encoding = "utf8"
  369. chunks.append((binary, encoding))
  370. # Turn our chunks back into a Header object, then let that
  371. # Header object do the right thing to turn them into a
  372. # string for us.
  373. value.append(str(email.header.make_header(chunks)))
  374. # This is already a string, so just add it.
  375. else:
  376. value.append(h)
  377. # We've processed all of our values to get them into a list of str,
  378. # but we may have mojibake data, in which case this is an unparsed
  379. # field.
  380. if not valid_encoding:
  381. unparsed[name] = value
  382. continue
  383. raw_name = _EMAIL_TO_RAW_MAPPING.get(name)
  384. if raw_name is None:
  385. # This is a bit of a weird situation, we've encountered a key that
  386. # we don't know what it means, so we don't know whether it's meant
  387. # to be a list or not.
  388. #
  389. # Since we can't really tell one way or another, we'll just leave it
  390. # as a list, even though it may be a single item list, because that's
  391. # what makes the most sense for email headers.
  392. unparsed[name] = value
  393. continue
  394. # If this is one of our string fields, then we'll check to see if our
  395. # value is a list of a single item. If it is then we'll assume that
  396. # it was emitted as a single string, and unwrap the str from inside
  397. # the list.
  398. #
  399. # If it's any other kind of data, then we haven't the faintest clue
  400. # what we should parse it as, and we have to just add it to our list
  401. # of unparsed stuff.
  402. if raw_name in _STRING_FIELDS and len(value) == 1:
  403. raw[raw_name] = value[0]
  404. # If this is import_names, we need to special case the empty field
  405. # case, which converts to an empty list instead of None. We can't let
  406. # the empty case slip through, as it will fail validation.
  407. elif raw_name == "import_names" and value == [""]:
  408. raw[raw_name] = []
  409. # If this is one of our list of string fields, then we can just assign
  410. # the value, since email *only* has strings, and our get_all() call
  411. # above ensures that this is a list.
  412. elif raw_name in _LIST_FIELDS:
  413. raw[raw_name] = value
  414. # Special Case: Keywords
  415. # The keywords field is implemented in the metadata spec as a str,
  416. # but it conceptually is a list of strings, and is serialized using
  417. # ", ".join(keywords), so we'll do some light data massaging to turn
  418. # this into what it logically is.
  419. elif raw_name == "keywords" and len(value) == 1:
  420. raw[raw_name] = _parse_keywords(value[0])
  421. # Special Case: Project-URL
  422. # The project urls is implemented in the metadata spec as a list of
  423. # specially-formatted strings that represent a key and a value, which
  424. # is fundamentally a mapping, however the email format doesn't support
  425. # mappings in a sane way, so it was crammed into a list of strings
  426. # instead.
  427. #
  428. # We will do a little light data massaging to turn this into a map as
  429. # it logically should be.
  430. elif raw_name == "project_urls":
  431. try:
  432. raw[raw_name] = _parse_project_urls(value)
  433. except KeyError:
  434. unparsed[name] = value
  435. # Nothing that we've done has managed to parse this, so it'll just
  436. # throw it in our unparsable data and move on.
  437. else:
  438. unparsed[name] = value
  439. # We need to support getting the Description from the message payload in
  440. # addition to getting it from the the headers. This does mean, though, there
  441. # is the possibility of it being set both ways, in which case we put both
  442. # in 'unparsed' since we don't know which is right.
  443. try:
  444. payload = _get_payload(parsed, data)
  445. except ValueError:
  446. unparsed.setdefault("description", []).append(
  447. parsed.get_payload(decode=isinstance(data, bytes)) # type: ignore[call-overload]
  448. )
  449. else:
  450. if payload:
  451. # Check to see if we've already got a description, if so then both
  452. # it, and this body move to unparsable.
  453. if "description" in raw:
  454. description_header = cast("str", raw.pop("description"))
  455. unparsed.setdefault("description", []).extend(
  456. [description_header, payload]
  457. )
  458. elif "description" in unparsed:
  459. unparsed["description"].append(payload)
  460. else:
  461. raw["description"] = payload
  462. # We need to cast our `raw` to a metadata, because a TypedDict only support
  463. # literal key names, but we're computing our key names on purpose, but the
  464. # way this function is implemented, our `TypedDict` can only have valid key
  465. # names.
  466. return cast("RawMetadata", raw), unparsed
  467. _NOT_FOUND = object()
  468. # Keep the two values in sync.
  469. _VALID_METADATA_VERSIONS = [
  470. "1.0",
  471. "1.1",
  472. "1.2",
  473. "2.1",
  474. "2.2",
  475. "2.3",
  476. "2.4",
  477. "2.5",
  478. "2.6",
  479. ]
  480. _MetadataVersion = Literal[
  481. "1.0", "1.1", "1.2", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6"
  482. ]
  483. _REQUIRED_ATTRS = frozenset(["metadata_version", "name", "version"])
  484. class _Validator(Generic[T]):
  485. """Validate a metadata field.
  486. All _process_*() methods correspond to a core metadata field. The method is
  487. called with the field's raw value. If the raw value is valid it is returned
  488. in its "enriched" form (e.g. ``version.Version`` for the ``Version`` field).
  489. If the raw value is invalid, :exc:`InvalidMetadata` is raised (with a cause
  490. as appropriate).
  491. """
  492. name: str
  493. raw_name: str
  494. added: _MetadataVersion
  495. def __init__(
  496. self,
  497. *,
  498. added: _MetadataVersion = "1.0",
  499. ) -> None:
  500. self.added = added
  501. def __set_name__(self, _owner: Metadata, name: str) -> None:
  502. self.name = name
  503. self.raw_name = _RAW_TO_EMAIL_MAPPING[name]
  504. def __get__(self, instance: Metadata, _owner: type[Metadata]) -> T:
  505. # With Python 3.8, the caching can be replaced with functools.cached_property().
  506. # No need to check the cache as attribute lookup will resolve into the
  507. # instance's __dict__ before __get__ is called.
  508. cache = instance.__dict__
  509. value = instance._raw.get(self.name)
  510. # To make the _process_* methods easier, we'll check if the value is None
  511. # and if this field is NOT a required attribute, and if both of those
  512. # things are true, we'll skip the the converter. This will mean that the
  513. # converters never have to deal with the None union.
  514. if self.name in _REQUIRED_ATTRS or value is not None:
  515. try:
  516. converter: Callable[[Any], T] = getattr(self, f"_process_{self.name}")
  517. except AttributeError:
  518. pass
  519. else:
  520. value = converter(value)
  521. cache[self.name] = value
  522. try:
  523. del instance._raw[self.name] # type: ignore[misc]
  524. except KeyError:
  525. pass
  526. return cast("T", value)
  527. def _invalid_metadata(
  528. self, msg: str, cause: Exception | None = None
  529. ) -> InvalidMetadata:
  530. exc = InvalidMetadata(self.raw_name, msg)
  531. exc.__cause__ = cause
  532. return exc
  533. def _process_metadata_version(self, value: str) -> _MetadataVersion:
  534. # Implicitly makes Metadata-Version required.
  535. if value not in _VALID_METADATA_VERSIONS:
  536. raise self._invalid_metadata(f"{value!r} is not a valid metadata version")
  537. return cast("_MetadataVersion", value)
  538. def _process_name(self, value: str) -> str:
  539. if not value:
  540. raise self._invalid_metadata(f"{self.raw_name!r} is a required field")
  541. # Validate the name as a side-effect.
  542. try:
  543. utils.canonicalize_name(value, validate=True)
  544. except utils.InvalidName as exc:
  545. raise self._invalid_metadata(
  546. f"{value!r} is invalid for {self.raw_name!r}", cause=exc
  547. ) from exc
  548. else:
  549. return value
  550. def _process_version(self, value: str) -> Version:
  551. if not value:
  552. raise self._invalid_metadata(f"{self.raw_name!r} is a required field")
  553. try:
  554. return version_module.parse(value)
  555. except version_module.InvalidVersion as exc:
  556. raise self._invalid_metadata(
  557. f"{value!r} is invalid for {self.raw_name!r}", cause=exc
  558. ) from exc
  559. def _process_summary(self, value: str) -> str:
  560. """Check the field contains no line breaks."""
  561. if _LINE_BOUNDARY_RE.search(value):
  562. raise self._invalid_metadata(f"{self.raw_name!r} must be a single line")
  563. return value
  564. def _process_description_content_type(self, value: str) -> str:
  565. content_types = {"text/plain", "text/x-rst", "text/markdown"}
  566. invalid_msg = (
  567. f"{self.raw_name!r} must be one of {list(content_types)}, not {value!r}"
  568. )
  569. message = email.message.EmailMessage()
  570. try:
  571. message["content-type"] = value
  572. # The email parser can raise IndexError on malformed RFC 2231
  573. # parameters such as "text/plain; x*".
  574. except (ValueError, IndexError) as exc:
  575. msg = f"{value!r} is not a valid content type for {self.raw_name!r}"
  576. raise self._invalid_metadata(msg, cause=exc) from exc
  577. content_type_header = message["content-type"]
  578. if content_type_header.defects:
  579. defect = content_type_header.defects[0]
  580. msg = (
  581. f"{value!r} is not a valid content type for {self.raw_name!r}: {defect}"
  582. )
  583. raise self._invalid_metadata(msg, cause=defect) from defect
  584. content_type, parameters = (
  585. # Defaults to `text/plain` if parsing failed.
  586. message.get_content_type().lower(),
  587. content_type_header.params,
  588. )
  589. # Check if content-type is valid or defaulted to `text/plain` and thus was
  590. # not parseable.
  591. if content_type not in content_types or content_type not in value.lower():
  592. raise self._invalid_metadata(invalid_msg)
  593. charset = parameters.get("charset", "UTF-8")
  594. if charset.lower() != "utf-8":
  595. raise self._invalid_metadata(
  596. f"{self.raw_name!r} can only specify the UTF-8 charset, not {charset!r}"
  597. )
  598. markdown_variants = {"GFM", "CommonMark"}
  599. variant = parameters.get("variant", "GFM") # Use an acceptable default.
  600. if content_type == "text/markdown" and variant not in markdown_variants:
  601. raise self._invalid_metadata(
  602. f"valid Markdown variants for {self.raw_name!r} are "
  603. f"{list(markdown_variants)}, not {variant!r}",
  604. )
  605. return value
  606. def _process_dynamic(self, value: list[str]) -> list[str]:
  607. dynamic_fields = list(map(str.lower, value))
  608. for dynamic_field in dynamic_fields:
  609. if dynamic_field in {"name", "version", "metadata-version"}:
  610. raise self._invalid_metadata(
  611. f"{dynamic_field!r} is not allowed as a dynamic field"
  612. )
  613. elif dynamic_field not in _EMAIL_TO_RAW_MAPPING:
  614. raise self._invalid_metadata(
  615. f"{dynamic_field!r} is not a valid dynamic field"
  616. )
  617. return dynamic_fields
  618. def _process_provides_extra(
  619. self,
  620. value: list[str],
  621. ) -> list[utils.NormalizedName]:
  622. normalized_names = []
  623. try:
  624. for name in value:
  625. normalized_names.append(utils.canonicalize_name(name, validate=True))
  626. except utils.InvalidName as exc:
  627. raise self._invalid_metadata(
  628. f"{name!r} is invalid for {self.raw_name!r}", cause=exc
  629. ) from exc
  630. else:
  631. return normalized_names
  632. def _process_requires_python(self, value: str) -> specifiers.SpecifierSet:
  633. try:
  634. return specifiers.SpecifierSet(value)
  635. except specifiers.InvalidSpecifier as exc:
  636. raise self._invalid_metadata(
  637. f"{value!r} is invalid for {self.raw_name!r}", cause=exc
  638. ) from exc
  639. def _process_requires_dist(
  640. self,
  641. value: list[str],
  642. ) -> list[requirements.Requirement]:
  643. reqs = []
  644. try:
  645. for req in value:
  646. reqs.append(requirements.Requirement(req))
  647. except requirements.InvalidRequirement as exc:
  648. raise self._invalid_metadata(
  649. f"{req!r} is invalid for {self.raw_name!r}", cause=exc
  650. ) from exc
  651. else:
  652. return reqs
  653. def _process_license_expression(self, value: str) -> NormalizedLicenseExpression:
  654. try:
  655. return licenses.canonicalize_license_expression(value)
  656. except ValueError as exc:
  657. raise self._invalid_metadata(
  658. f"{value!r} is invalid for {self.raw_name!r}", cause=exc
  659. ) from exc
  660. def _process_license_files(self, value: list[str]) -> list[str]:
  661. paths = []
  662. for path in value:
  663. if ".." in path:
  664. raise self._invalid_metadata(
  665. f"{path!r} is invalid for {self.raw_name!r}, "
  666. "parent directory indicators are not allowed"
  667. )
  668. if "*" in path:
  669. raise self._invalid_metadata(
  670. f"{path!r} is invalid for {self.raw_name!r}, paths must be resolved"
  671. )
  672. if (
  673. pathlib.PurePosixPath(path).is_absolute()
  674. or pathlib.PureWindowsPath(path).is_absolute()
  675. ):
  676. raise self._invalid_metadata(
  677. f"{path!r} is invalid for {self.raw_name!r}, paths must be relative"
  678. )
  679. if pathlib.PureWindowsPath(path).as_posix() != path:
  680. raise self._invalid_metadata(
  681. f"{path!r} is invalid for {self.raw_name!r}, "
  682. "paths must use '/' delimiter"
  683. )
  684. paths.append(path)
  685. return paths
  686. def _process_import_names(self, value: list[str]) -> list[str]:
  687. for import_name in value:
  688. name, semicolon, private = import_name.partition(";")
  689. name = name.rstrip()
  690. for identifier in name.split("."):
  691. if not identifier.isidentifier():
  692. raise self._invalid_metadata(
  693. f"{name!r} is invalid for {self.raw_name!r}; "
  694. f"{identifier!r} is not a valid identifier"
  695. )
  696. elif keyword.iskeyword(identifier):
  697. raise self._invalid_metadata(
  698. f"{name!r} is invalid for {self.raw_name!r}; "
  699. f"{identifier!r} is a keyword"
  700. )
  701. if semicolon and private.lstrip() != "private":
  702. raise self._invalid_metadata(
  703. f"{import_name!r} is invalid for {self.raw_name!r}; "
  704. "the only valid option is 'private'"
  705. )
  706. return value
  707. _process_import_namespaces = _process_import_names
  708. class Metadata:
  709. """Representation of distribution metadata.
  710. Compared to :class:`RawMetadata`, this class provides objects representing
  711. metadata fields instead of only using built-in types. Any invalid metadata
  712. will cause :exc:`InvalidMetadata` to be raised (with a
  713. :py:attr:`~BaseException.__cause__` attribute as appropriate).
  714. .. versionadded:: 23.2
  715. .. versionchanged:: 24.0
  716. Optional attributes now return None when the field is absent instead of
  717. raising.
  718. """
  719. _raw: RawMetadata
  720. @classmethod
  721. def from_raw(cls, data: RawMetadata, *, validate: bool = True) -> Metadata:
  722. """Create an instance from :class:`RawMetadata`.
  723. If *validate* is true, all metadata will be validated and all related
  724. exceptions will be gathered and raised as an :class:`ExceptionGroup`;
  725. otherwise, validation happens per attribute when it is accessed.
  726. """
  727. ins = cls()
  728. ins._raw = data.copy() # Mutations occur due to caching enriched values.
  729. if validate:
  730. collector = _ErrorCollector()
  731. metadata_version = None
  732. with collector.collect(InvalidMetadata):
  733. metadata_version = ins.metadata_version
  734. metadata_age = _VALID_METADATA_VERSIONS.index(metadata_version)
  735. # Make sure to check for the fields that are present, the required
  736. # fields (so their absence can be reported).
  737. fields_to_check = frozenset(ins._raw) | _REQUIRED_ATTRS
  738. # Remove fields that have already been checked.
  739. fields_to_check -= {"metadata_version"}
  740. for key in fields_to_check:
  741. try:
  742. if metadata_version:
  743. # Can't use getattr() as that triggers descriptor protocol which
  744. # will fail due to no value for the instance argument.
  745. try:
  746. field_metadata_version = cls.__dict__[key].added
  747. except KeyError:
  748. exc = InvalidMetadata(key, f"unrecognized field: {key!r}")
  749. collector.error(exc)
  750. continue
  751. field_age = _VALID_METADATA_VERSIONS.index(
  752. field_metadata_version
  753. )
  754. if field_age > metadata_age:
  755. field = _RAW_TO_EMAIL_MAPPING[key]
  756. exc = InvalidMetadata(
  757. field,
  758. f"{field} introduced in metadata version "
  759. f"{field_metadata_version}, not {metadata_version}",
  760. )
  761. collector.error(exc)
  762. continue
  763. getattr(ins, key)
  764. except InvalidMetadata as exc:
  765. collector.error(exc)
  766. collector.finalize("invalid metadata")
  767. return ins
  768. @classmethod
  769. def from_email(cls, data: bytes | str, *, validate: bool = True) -> Metadata:
  770. """Parse metadata from email headers.
  771. If *validate* is true, the metadata will be validated. All exceptions
  772. related to validation will be gathered and raised as an :class:`ExceptionGroup`.
  773. """
  774. raw, unparsed = parse_email(data)
  775. if validate:
  776. with _ErrorCollector().on_exit("invalid or unparsed metadata") as collector:
  777. for unparsed_key in unparsed:
  778. if unparsed_key in _EMAIL_TO_RAW_MAPPING:
  779. message = f"{unparsed_key!r} has invalid data"
  780. else:
  781. message = f"unrecognized field: {unparsed_key!r}"
  782. collector.error(InvalidMetadata(unparsed_key, message))
  783. try:
  784. validated = cls.from_raw(raw, validate=validate)
  785. except ExceptionGroup as exc_group:
  786. # The no-branch pragmas cover arcs only seen by Python 3.9.
  787. for exc in exc_group.exceptions: # pragma: no branch
  788. # A required field reported above as unparsed is absent
  789. # from `raw`, so skip from_raw's duplicate "missing"
  790. # complaint.
  791. if not (
  792. isinstance(exc, InvalidMetadata)
  793. and exc.field in unparsed
  794. and _EMAIL_TO_RAW_MAPPING.get(exc.field) not in raw
  795. ):
  796. collector.error(exc)
  797. else:
  798. if not collector.errors: # pragma: no branch
  799. return validated
  800. return cls.from_raw(raw, validate=validate)
  801. metadata_version: _Validator[_MetadataVersion] = _Validator()
  802. """:external:ref:`core-metadata-metadata-version`
  803. (required; validated to be a valid metadata version)"""
  804. # `name` is not normalized/typed to NormalizedName so as to provide access to
  805. # the original/raw name.
  806. name: _Validator[str] = _Validator()
  807. """:external:ref:`core-metadata-name`
  808. (required; validated using :func:`~packaging.utils.canonicalize_name` and its
  809. *validate* parameter)"""
  810. version: _Validator[Version] = _Validator()
  811. """:external:ref:`core-metadata-version` (required)"""
  812. dynamic: _Validator[list[str] | None] = _Validator(
  813. added="2.2",
  814. )
  815. """:external:ref:`core-metadata-dynamic`
  816. (validated against core metadata field names and lowercased)"""
  817. platforms: _Validator[list[str] | None] = _Validator()
  818. """:external:ref:`core-metadata-platform`"""
  819. supported_platforms: _Validator[list[str] | None] = _Validator(added="1.1")
  820. """:external:ref:`core-metadata-supported-platform`"""
  821. summary: _Validator[str | None] = _Validator()
  822. """:external:ref:`core-metadata-summary` (validated to contain no newlines)"""
  823. description: _Validator[str | None] = _Validator() # TODO 2.1: can be in body
  824. """:external:ref:`core-metadata-description`"""
  825. description_content_type: _Validator[str | None] = _Validator(added="2.1")
  826. """:external:ref:`core-metadata-description-content-type` (validated)"""
  827. keywords: _Validator[list[str] | None] = _Validator()
  828. """:external:ref:`core-metadata-keywords`"""
  829. home_page: _Validator[str | None] = _Validator()
  830. """:external:ref:`core-metadata-home-page`"""
  831. download_url: _Validator[str | None] = _Validator(added="1.1")
  832. """:external:ref:`core-metadata-download-url`"""
  833. author: _Validator[str | None] = _Validator()
  834. """:external:ref:`core-metadata-author`"""
  835. author_email: _Validator[str | None] = _Validator()
  836. """:external:ref:`core-metadata-author-email`"""
  837. maintainer: _Validator[str | None] = _Validator(added="1.2")
  838. """:external:ref:`core-metadata-maintainer`"""
  839. maintainer_email: _Validator[str | None] = _Validator(added="1.2")
  840. """:external:ref:`core-metadata-maintainer-email`"""
  841. license: _Validator[str | None] = _Validator()
  842. """:external:ref:`core-metadata-license`"""
  843. license_expression: _Validator[NormalizedLicenseExpression | None] = _Validator(
  844. added="2.4"
  845. )
  846. """:external:ref:`core-metadata-license-expression`
  847. .. versionadded:: 24.2
  848. """
  849. license_files: _Validator[list[str] | None] = _Validator(added="2.4")
  850. """:external:ref:`core-metadata-license-file`
  851. .. versionadded:: 24.2
  852. """
  853. classifiers: _Validator[list[str] | None] = _Validator(added="1.1")
  854. """:external:ref:`core-metadata-classifier`"""
  855. requires_dist: _Validator[list[requirements.Requirement] | None] = _Validator(
  856. added="1.2"
  857. )
  858. """:external:ref:`core-metadata-requires-dist`"""
  859. requires_python: _Validator[specifiers.SpecifierSet | None] = _Validator(
  860. added="1.2"
  861. )
  862. """:external:ref:`core-metadata-requires-python`"""
  863. # Because `Requires-External` allows for non-PEP 440 version specifiers, we
  864. # don't do any processing on the values.
  865. requires_external: _Validator[list[str] | None] = _Validator(added="1.2")
  866. """:external:ref:`core-metadata-requires-external`"""
  867. project_urls: _Validator[dict[str, str] | None] = _Validator(added="1.2")
  868. """:external:ref:`core-metadata-project-url`"""
  869. # PEP 685 lets us raise an error if an extra doesn't pass `Name` validation
  870. # regardless of metadata version.
  871. provides_extra: _Validator[list[utils.NormalizedName] | None] = _Validator(
  872. added="2.1",
  873. )
  874. """:external:ref:`core-metadata-provides-extra`"""
  875. provides_dist: _Validator[list[str] | None] = _Validator(added="1.2")
  876. """:external:ref:`core-metadata-provides-dist`"""
  877. obsoletes_dist: _Validator[list[str] | None] = _Validator(added="1.2")
  878. """:external:ref:`core-metadata-obsoletes-dist`"""
  879. import_names: _Validator[list[str] | None] = _Validator(added="2.5")
  880. """:external:ref:`core-metadata-import-name`
  881. .. versionadded:: 26.0
  882. """
  883. import_namespaces: _Validator[list[str] | None] = _Validator(added="2.5")
  884. """:external:ref:`core-metadata-import-namespace`
  885. .. versionadded:: 26.0
  886. """
  887. requires: _Validator[list[str] | None] = _Validator(added="1.1")
  888. """``Requires`` (deprecated)"""
  889. provides: _Validator[list[str] | None] = _Validator(added="1.1")
  890. """``Provides`` (deprecated)"""
  891. obsoletes: _Validator[list[str] | None] = _Validator(added="1.1")
  892. """``Obsoletes`` (deprecated)"""
  893. def as_rfc822(self) -> RFC822Message:
  894. """
  895. Return an RFC822 message with the metadata.
  896. .. versionadded:: 26.0
  897. """
  898. message = RFC822Message()
  899. self._write_metadata(message)
  900. return message
  901. def _write_metadata(self, message: RFC822Message) -> None:
  902. """
  903. Return an RFC822 message with the metadata.
  904. """
  905. for name, validator in self.__class__.__dict__.items():
  906. if isinstance(validator, _Validator) and name != "description":
  907. value = getattr(self, name)
  908. email_name = _RAW_TO_EMAIL_MAPPING[name]
  909. if value is not None:
  910. if email_name == "project-url":
  911. for label, url in value.items():
  912. message[email_name] = f"{label}, {url}"
  913. elif email_name == "keywords":
  914. message[email_name] = ",".join(value)
  915. elif email_name == "import-name" and value == []:
  916. message[email_name] = ""
  917. elif isinstance(value, list):
  918. for item in value:
  919. message[email_name] = str(item)
  920. else:
  921. message[email_name] = str(value)
  922. # The description is a special case because it is in the body of the message.
  923. if self.description is not None:
  924. message.set_payload(self.description)