_parser.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793
  1. # SPDX-License-Identifier: MIT
  2. # SPDX-FileCopyrightText: 2021 Taneli Hukkinen
  3. # Licensed to PSF under a Contributor Agreement.
  4. from __future__ import annotations
  5. import sys
  6. from types import MappingProxyType
  7. from ._re import (
  8. RE_DATETIME,
  9. RE_LOCALTIME,
  10. RE_NUMBER,
  11. match_to_datetime,
  12. match_to_localtime,
  13. match_to_number,
  14. )
  15. TYPE_CHECKING = False
  16. if TYPE_CHECKING:
  17. from collections.abc import Iterable
  18. from typing import IO, Any, Final
  19. from ._types import Key, ParseFloat, Pos
  20. # Inline tables/arrays are implemented using recursion. Pathologically
  21. # nested documents cause pure Python to raise RecursionError (which is OK),
  22. # but mypyc binary wheels will crash unrecoverably (not OK). According to
  23. # mypyc docs this will be fixed in the future:
  24. # https://mypyc.readthedocs.io/en/latest/differences_from_python.html#stack-overflows
  25. # Before mypyc's fix is in, recursion needs to be limited by this library.
  26. # Choosing `sys.getrecursionlimit()` as maximum inline table/array nesting
  27. # level, as it allows more nesting than pure Python, but still seems a far
  28. # lower number than where mypyc binaries crash.
  29. MAX_INLINE_NESTING: Final = sys.getrecursionlimit()
  30. # Pathologically excessive number of parts in a key runs into quadratic
  31. # behavior (e.g. in Flags.is_).
  32. # Even if keys aren't currently parsed using recursion, they name a
  33. # recursive structure, so it makes sense to limit it using getrecursionlimit()
  34. # and RecursionError.
  35. MAX_KEY_PARTS: Final = sys.getrecursionlimit()
  36. ASCII_CTRL: Final = frozenset(chr(i) for i in range(32)) | frozenset(chr(127))
  37. # Neither of these sets include quotation mark or backslash. They are
  38. # currently handled as separate cases in the parser functions.
  39. ILLEGAL_BASIC_STR_CHARS: Final = ASCII_CTRL - frozenset("\t")
  40. ILLEGAL_MULTILINE_BASIC_STR_CHARS: Final = ASCII_CTRL - frozenset("\t\n")
  41. ILLEGAL_LITERAL_STR_CHARS: Final = ILLEGAL_BASIC_STR_CHARS
  42. ILLEGAL_MULTILINE_LITERAL_STR_CHARS: Final = ILLEGAL_MULTILINE_BASIC_STR_CHARS
  43. ILLEGAL_COMMENT_CHARS: Final = ILLEGAL_BASIC_STR_CHARS
  44. TOML_WS: Final = frozenset(" \t")
  45. TOML_WS_AND_NEWLINE: Final = TOML_WS | frozenset("\n")
  46. BARE_KEY_CHARS: Final = frozenset(
  47. "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789" "-_"
  48. )
  49. KEY_INITIAL_CHARS: Final = BARE_KEY_CHARS | frozenset("\"'")
  50. HEXDIGIT_CHARS: Final = frozenset("abcdef" "ABCDEF" "0123456789")
  51. BASIC_STR_ESCAPE_REPLACEMENTS: Final = MappingProxyType(
  52. {
  53. "\\b": "\u0008", # backspace
  54. "\\t": "\u0009", # tab
  55. "\\n": "\u000a", # linefeed
  56. "\\f": "\u000c", # form feed
  57. "\\r": "\u000d", # carriage return
  58. "\\e": "\u001b", # escape
  59. '\\"': "\u0022", # quote
  60. "\\\\": "\u005c", # backslash
  61. }
  62. )
  63. class DEPRECATED_DEFAULT:
  64. """Sentinel to be used as default arg during deprecation
  65. period of TOMLDecodeError's free-form arguments."""
  66. class TOMLDecodeError(ValueError):
  67. """An error raised if a document is not valid TOML.
  68. Adds the following attributes to ValueError:
  69. msg: The unformatted error message
  70. doc: The TOML document being parsed
  71. pos: The index of doc where parsing failed
  72. lineno: The line corresponding to pos
  73. colno: The column corresponding to pos
  74. """
  75. def __init__(
  76. self,
  77. msg: str | type[DEPRECATED_DEFAULT] = DEPRECATED_DEFAULT,
  78. doc: str | type[DEPRECATED_DEFAULT] = DEPRECATED_DEFAULT,
  79. pos: Pos | type[DEPRECATED_DEFAULT] = DEPRECATED_DEFAULT,
  80. *args: Any,
  81. ):
  82. if (
  83. args
  84. or not isinstance(msg, str)
  85. or not isinstance(doc, str)
  86. or not isinstance(pos, int)
  87. ):
  88. import warnings
  89. warnings.warn(
  90. "Free-form arguments for TOMLDecodeError are deprecated. "
  91. "Please set 'msg' (str), 'doc' (str) and 'pos' (int) arguments only.",
  92. DeprecationWarning,
  93. stacklevel=2,
  94. )
  95. if pos is not DEPRECATED_DEFAULT:
  96. args = pos, *args
  97. if doc is not DEPRECATED_DEFAULT:
  98. args = doc, *args
  99. if msg is not DEPRECATED_DEFAULT:
  100. args = msg, *args
  101. ValueError.__init__(self, *args)
  102. return
  103. lineno = doc.count("\n", 0, pos) + 1
  104. if lineno == 1:
  105. colno = pos + 1
  106. else:
  107. colno = pos - doc.rindex("\n", 0, pos)
  108. if pos >= len(doc):
  109. coord_repr = "end of document"
  110. else:
  111. coord_repr = f"line {lineno}, column {colno}"
  112. errmsg = f"{msg} (at {coord_repr})"
  113. ValueError.__init__(self, errmsg)
  114. self.msg = msg
  115. self.doc = doc
  116. self.pos = pos
  117. self.lineno = lineno
  118. self.colno = colno
  119. def load(__fp: IO[bytes], *, parse_float: ParseFloat = float) -> dict[str, Any]:
  120. """Parse TOML from a binary file object."""
  121. b = __fp.read()
  122. try:
  123. s = b.decode()
  124. except AttributeError:
  125. raise TypeError(
  126. "File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`"
  127. ) from None
  128. return loads(s, parse_float=parse_float)
  129. def loads(__s: str, *, parse_float: ParseFloat = float) -> dict[str, Any]:
  130. """Parse TOML from a string."""
  131. # The spec allows converting "\r\n" to "\n", even in string
  132. # literals. Let's do so to simplify parsing.
  133. try:
  134. src = __s.replace("\r\n", "\n")
  135. except (AttributeError, TypeError):
  136. raise TypeError(
  137. f"Expected str object, not '{type(__s).__qualname__}'"
  138. ) from None
  139. pos = 0
  140. out = Output()
  141. header: Key = ()
  142. parse_float = make_safe_parse_float(parse_float)
  143. # Parse one statement at a time
  144. # (typically means one line in TOML source)
  145. while True:
  146. # 1. Skip line leading whitespace
  147. pos = skip_chars(src, pos, TOML_WS)
  148. # 2. Parse rules. Expect one of the following:
  149. # - end of file
  150. # - end of line
  151. # - comment
  152. # - key/value pair
  153. # - append dict to list (and move to its namespace)
  154. # - create dict (and move to its namespace)
  155. # Skip trailing whitespace when applicable.
  156. try:
  157. char = src[pos]
  158. except IndexError:
  159. break
  160. if char == "\n":
  161. pos += 1
  162. continue
  163. if char in KEY_INITIAL_CHARS:
  164. pos = key_value_rule(src, pos, out, header, parse_float)
  165. pos = skip_chars(src, pos, TOML_WS)
  166. elif char == "[":
  167. try:
  168. second_char: str | None = src[pos + 1]
  169. except IndexError:
  170. second_char = None
  171. out.flags.finalize_pending()
  172. if second_char == "[":
  173. pos, header = create_list_rule(src, pos, out)
  174. else:
  175. pos, header = create_dict_rule(src, pos, out)
  176. pos = skip_chars(src, pos, TOML_WS)
  177. elif char != "#":
  178. raise TOMLDecodeError("Invalid statement", src, pos)
  179. # 3. Skip comment
  180. pos = skip_comment(src, pos)
  181. # 4. Expect end of line or end of file
  182. try:
  183. char = src[pos]
  184. except IndexError:
  185. break
  186. if char != "\n":
  187. raise TOMLDecodeError(
  188. "Expected newline or end of document after a statement", src, pos
  189. )
  190. pos += 1
  191. return out.data.dict
  192. class Flags:
  193. """Flags that map to parsed keys/namespaces."""
  194. # Marks an immutable namespace (inline array or inline table).
  195. FROZEN: Final = 0
  196. # Marks a nest that has been explicitly created and can no longer
  197. # be opened using the "[table]" syntax.
  198. EXPLICIT_NEST: Final = 1
  199. def __init__(self) -> None:
  200. self._flags: dict[str, dict[Any, Any]] = {}
  201. self._pending_flags: set[tuple[Key, int]] = set()
  202. def add_pending(self, key: Key, flag: int) -> None:
  203. self._pending_flags.add((key, flag))
  204. def finalize_pending(self) -> None:
  205. for key, flag in self._pending_flags:
  206. self.set(key, flag, recursive=False)
  207. self._pending_flags.clear()
  208. def unset_all(self, key: Key) -> None:
  209. cont = self._flags
  210. for k in key[:-1]:
  211. if k not in cont:
  212. return
  213. cont = cont[k]["nested"]
  214. cont.pop(key[-1], None)
  215. def set(self, key: Key, flag: int, *, recursive: bool) -> None: # noqa: A003
  216. cont = self._flags
  217. key_parent, key_stem = key[:-1], key[-1]
  218. for k in key_parent:
  219. if k not in cont:
  220. cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}}
  221. cont = cont[k]["nested"]
  222. if key_stem not in cont:
  223. cont[key_stem] = {"flags": set(), "recursive_flags": set(), "nested": {}}
  224. cont[key_stem]["recursive_flags" if recursive else "flags"].add(flag)
  225. def is_(self, key: Key, flag: int) -> bool:
  226. if not key:
  227. return False # document root has no flags
  228. cont = self._flags
  229. for k in key[:-1]:
  230. if k not in cont:
  231. return False
  232. inner_cont = cont[k]
  233. if flag in inner_cont["recursive_flags"]:
  234. return True
  235. cont = inner_cont["nested"]
  236. key_stem = key[-1]
  237. if key_stem in cont:
  238. inner_cont = cont[key_stem]
  239. return flag in inner_cont["flags"] or flag in inner_cont["recursive_flags"]
  240. return False
  241. class NestedDict:
  242. def __init__(self) -> None:
  243. # The parsed content of the TOML document
  244. self.dict: dict[str, Any] = {}
  245. def get_or_create_nest(
  246. self,
  247. key: Key,
  248. *,
  249. access_lists: bool = True,
  250. ) -> dict[str, Any]:
  251. cont: Any = self.dict
  252. for k in key:
  253. if k not in cont:
  254. cont[k] = {}
  255. cont = cont[k]
  256. if access_lists and isinstance(cont, list):
  257. cont = cont[-1]
  258. if not isinstance(cont, dict):
  259. raise KeyError("There is no nest behind this key")
  260. return cont # type: ignore[no-any-return]
  261. def append_nest_to_list(self, key: Key) -> None:
  262. cont = self.get_or_create_nest(key[:-1])
  263. last_key = key[-1]
  264. if last_key in cont:
  265. list_ = cont[last_key]
  266. if not isinstance(list_, list):
  267. raise KeyError("An object other than list found behind this key")
  268. list_.append({})
  269. else:
  270. cont[last_key] = [{}]
  271. class Output:
  272. def __init__(self) -> None:
  273. self.data = NestedDict()
  274. self.flags = Flags()
  275. def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos:
  276. try:
  277. while src[pos] in chars:
  278. pos += 1
  279. except IndexError:
  280. pass
  281. return pos
  282. def skip_until(
  283. src: str,
  284. pos: Pos,
  285. expect: str,
  286. *,
  287. error_on: frozenset[str],
  288. error_on_eof: bool,
  289. ) -> Pos:
  290. try:
  291. new_pos = src.index(expect, pos)
  292. except ValueError:
  293. new_pos = len(src)
  294. if error_on_eof:
  295. raise TOMLDecodeError(f"Expected {expect!r}", src, new_pos) from None
  296. if not error_on.isdisjoint(src[pos:new_pos]):
  297. while src[pos] not in error_on:
  298. pos += 1
  299. raise TOMLDecodeError(f"Found invalid character {src[pos]!r}", src, pos)
  300. return new_pos
  301. def skip_comment(src: str, pos: Pos) -> Pos:
  302. try:
  303. char: str | None = src[pos]
  304. except IndexError:
  305. char = None
  306. if char == "#":
  307. return skip_until(
  308. src, pos + 1, "\n", error_on=ILLEGAL_COMMENT_CHARS, error_on_eof=False
  309. )
  310. return pos
  311. def skip_comments_and_array_ws(src: str, pos: Pos) -> Pos:
  312. while True:
  313. pos_before_skip = pos
  314. pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE)
  315. pos = skip_comment(src, pos)
  316. if pos == pos_before_skip:
  317. return pos
  318. def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]:
  319. pos += 1 # Skip "["
  320. pos = skip_chars(src, pos, TOML_WS)
  321. pos, key = parse_key(src, pos)
  322. if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN):
  323. raise TOMLDecodeError(f"Cannot declare {key} twice", src, pos)
  324. out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False)
  325. try:
  326. out.data.get_or_create_nest(key)
  327. except KeyError:
  328. raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None
  329. if not src.startswith("]", pos):
  330. raise TOMLDecodeError(
  331. "Expected ']' at the end of a table declaration", src, pos
  332. )
  333. return pos + 1, key
  334. def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]:
  335. pos += 2 # Skip "[["
  336. pos = skip_chars(src, pos, TOML_WS)
  337. pos, key = parse_key(src, pos)
  338. if out.flags.is_(key, Flags.FROZEN):
  339. raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos)
  340. # Free the namespace now that it points to another empty list item...
  341. out.flags.unset_all(key)
  342. # ...but this key precisely is still prohibited from table declaration
  343. out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False)
  344. try:
  345. out.data.append_nest_to_list(key)
  346. except KeyError:
  347. raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None
  348. if not src.startswith("]]", pos):
  349. raise TOMLDecodeError(
  350. "Expected ']]' at the end of an array declaration", src, pos
  351. )
  352. return pos + 2, key
  353. def key_value_rule(
  354. src: str, pos: Pos, out: Output, header: Key, parse_float: ParseFloat
  355. ) -> Pos:
  356. pos, key, value = parse_key_value_pair(src, pos, parse_float, nest_lvl=0)
  357. key_parent, key_stem = key[:-1], key[-1]
  358. abs_key_parent = header + key_parent
  359. relative_path_cont_keys = (header + key[:i] for i in range(1, len(key)))
  360. for cont_key in relative_path_cont_keys:
  361. # Check that dotted key syntax does not redefine an existing table
  362. if out.flags.is_(cont_key, Flags.EXPLICIT_NEST):
  363. raise TOMLDecodeError(f"Cannot redefine namespace {cont_key}", src, pos)
  364. # Containers in the relative path can't be opened with the table syntax or
  365. # dotted key/value syntax in following table sections.
  366. out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST)
  367. if out.flags.is_(abs_key_parent, Flags.FROZEN):
  368. raise TOMLDecodeError(
  369. f"Cannot mutate immutable namespace {abs_key_parent}", src, pos
  370. )
  371. try:
  372. nest = out.data.get_or_create_nest(abs_key_parent)
  373. except KeyError:
  374. raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None
  375. if key_stem in nest:
  376. raise TOMLDecodeError("Cannot overwrite a value", src, pos)
  377. # Mark inline table and array namespaces recursively immutable
  378. if isinstance(value, (dict, list)):
  379. out.flags.set(header + key, Flags.FROZEN, recursive=True)
  380. nest[key_stem] = value
  381. return pos
  382. def parse_key_value_pair(
  383. src: str, pos: Pos, parse_float: ParseFloat, nest_lvl: int
  384. ) -> tuple[Pos, Key, Any]:
  385. pos, key = parse_key(src, pos)
  386. try:
  387. char: str | None = src[pos]
  388. except IndexError:
  389. char = None
  390. if char != "=":
  391. raise TOMLDecodeError("Expected '=' after a key in a key/value pair", src, pos)
  392. pos += 1
  393. pos = skip_chars(src, pos, TOML_WS)
  394. pos, value = parse_value(src, pos, parse_float, nest_lvl)
  395. return pos, key, value
  396. def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]:
  397. pos, key_part = parse_key_part(src, pos)
  398. key: Key = (key_part,)
  399. pos = skip_chars(src, pos, TOML_WS)
  400. while True:
  401. try:
  402. char: str | None = src[pos]
  403. except IndexError:
  404. char = None
  405. if char != ".":
  406. return pos, key
  407. pos += 1
  408. pos = skip_chars(src, pos, TOML_WS)
  409. pos, key_part = parse_key_part(src, pos)
  410. key += (key_part,)
  411. if len(key) > MAX_KEY_PARTS:
  412. raise RecursionError(
  413. f"TOML key has more than the allowed {MAX_KEY_PARTS} parts"
  414. )
  415. pos = skip_chars(src, pos, TOML_WS)
  416. def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]:
  417. try:
  418. char: str | None = src[pos]
  419. except IndexError:
  420. char = None
  421. if char in BARE_KEY_CHARS:
  422. start_pos = pos
  423. pos = skip_chars(src, pos, BARE_KEY_CHARS)
  424. return pos, src[start_pos:pos]
  425. if char == "'":
  426. return parse_literal_str(src, pos)
  427. if char == '"':
  428. return parse_one_line_basic_str(src, pos)
  429. raise TOMLDecodeError("Invalid initial character for a key part", src, pos)
  430. def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]:
  431. pos += 1
  432. return parse_basic_str(src, pos, multiline=False)
  433. def parse_array(
  434. src: str, pos: Pos, parse_float: ParseFloat, nest_lvl: int
  435. ) -> tuple[Pos, list[Any]]:
  436. pos += 1
  437. array: list[Any] = []
  438. pos = skip_comments_and_array_ws(src, pos)
  439. if src.startswith("]", pos):
  440. return pos + 1, array
  441. while True:
  442. pos, val = parse_value(src, pos, parse_float, nest_lvl)
  443. array.append(val)
  444. pos = skip_comments_and_array_ws(src, pos)
  445. c = src[pos : pos + 1]
  446. if c == "]":
  447. return pos + 1, array
  448. if c != ",":
  449. raise TOMLDecodeError("Unclosed array", src, pos)
  450. pos += 1
  451. pos = skip_comments_and_array_ws(src, pos)
  452. if src.startswith("]", pos):
  453. return pos + 1, array
  454. def parse_inline_table(
  455. src: str, pos: Pos, parse_float: ParseFloat, nest_lvl: int
  456. ) -> tuple[Pos, dict[str, Any]]:
  457. pos += 1
  458. nested_dict = NestedDict()
  459. flags = Flags()
  460. pos = skip_comments_and_array_ws(src, pos)
  461. if src.startswith("}", pos):
  462. return pos + 1, nested_dict.dict
  463. while True:
  464. pos, key, value = parse_key_value_pair(src, pos, parse_float, nest_lvl)
  465. key_parent, key_stem = key[:-1], key[-1]
  466. if flags.is_(key, Flags.FROZEN):
  467. raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos)
  468. try:
  469. nest = nested_dict.get_or_create_nest(key_parent, access_lists=False)
  470. except KeyError:
  471. raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None
  472. if key_stem in nest:
  473. raise TOMLDecodeError(f"Duplicate inline table key {key_stem!r}", src, pos)
  474. nest[key_stem] = value
  475. pos = skip_comments_and_array_ws(src, pos)
  476. c = src[pos : pos + 1]
  477. if c == "}":
  478. return pos + 1, nested_dict.dict
  479. if c != ",":
  480. raise TOMLDecodeError("Unclosed inline table", src, pos)
  481. pos += 1
  482. pos = skip_comments_and_array_ws(src, pos)
  483. if src.startswith("}", pos):
  484. return pos + 1, nested_dict.dict
  485. if isinstance(value, (dict, list)):
  486. flags.set(key, Flags.FROZEN, recursive=True)
  487. def parse_basic_str_escape(
  488. src: str, pos: Pos, *, multiline: bool = False
  489. ) -> tuple[Pos, str]:
  490. escape_id = src[pos : pos + 2]
  491. pos += 2
  492. if multiline and escape_id in {"\\ ", "\\\t", "\\\n"}:
  493. # Skip whitespace until next non-whitespace character or end of
  494. # the doc. Error if non-whitespace is found before newline.
  495. if escape_id != "\\\n":
  496. pos = skip_chars(src, pos, TOML_WS)
  497. try:
  498. char = src[pos]
  499. except IndexError:
  500. return pos, ""
  501. if char != "\n":
  502. raise TOMLDecodeError("Unescaped '\\' in a string", src, pos)
  503. pos += 1
  504. pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE)
  505. return pos, ""
  506. if escape_id == "\\x":
  507. return parse_hex_char(src, pos, 2)
  508. if escape_id == "\\u":
  509. return parse_hex_char(src, pos, 4)
  510. if escape_id == "\\U":
  511. return parse_hex_char(src, pos, 8)
  512. try:
  513. return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id]
  514. except KeyError:
  515. raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) from None
  516. def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]:
  517. return parse_basic_str_escape(src, pos, multiline=True)
  518. def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]:
  519. hex_str = src[pos : pos + hex_len]
  520. if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str):
  521. raise TOMLDecodeError("Invalid hex value", src, pos)
  522. pos += hex_len
  523. hex_int = int(hex_str, 16)
  524. if not is_unicode_scalar_value(hex_int):
  525. raise TOMLDecodeError(
  526. "Escaped character is not a Unicode scalar value", src, pos
  527. )
  528. return pos, chr(hex_int)
  529. def parse_literal_str(src: str, pos: Pos) -> tuple[Pos, str]:
  530. pos += 1 # Skip starting apostrophe
  531. start_pos = pos
  532. pos = skip_until(
  533. src, pos, "'", error_on=ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True
  534. )
  535. return pos + 1, src[start_pos:pos] # Skip ending apostrophe
  536. def parse_multiline_str(src: str, pos: Pos, *, literal: bool) -> tuple[Pos, str]:
  537. pos += 3
  538. if src.startswith("\n", pos):
  539. pos += 1
  540. if literal:
  541. delim = "'"
  542. end_pos = skip_until(
  543. src,
  544. pos,
  545. "'''",
  546. error_on=ILLEGAL_MULTILINE_LITERAL_STR_CHARS,
  547. error_on_eof=True,
  548. )
  549. result = src[pos:end_pos]
  550. pos = end_pos + 3
  551. else:
  552. delim = '"'
  553. pos, result = parse_basic_str(src, pos, multiline=True)
  554. # Add at maximum two extra apostrophes/quotes if the end sequence
  555. # is 4 or 5 chars long instead of just 3.
  556. if not src.startswith(delim, pos):
  557. return pos, result
  558. pos += 1
  559. if not src.startswith(delim, pos):
  560. return pos, result + delim
  561. pos += 1
  562. return pos, result + (delim * 2)
  563. def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]:
  564. if multiline:
  565. error_on = ILLEGAL_MULTILINE_BASIC_STR_CHARS
  566. parse_escapes = parse_basic_str_escape_multiline
  567. else:
  568. error_on = ILLEGAL_BASIC_STR_CHARS
  569. parse_escapes = parse_basic_str_escape
  570. result = ""
  571. start_pos = pos
  572. while True:
  573. try:
  574. char = src[pos]
  575. except IndexError:
  576. raise TOMLDecodeError("Unterminated string", src, pos) from None
  577. if char == '"':
  578. if not multiline:
  579. return pos + 1, result + src[start_pos:pos]
  580. if src.startswith('"""', pos):
  581. return pos + 3, result + src[start_pos:pos]
  582. pos += 1
  583. continue
  584. if char == "\\":
  585. result += src[start_pos:pos]
  586. pos, parsed_escape = parse_escapes(src, pos)
  587. result += parsed_escape
  588. start_pos = pos
  589. continue
  590. if char in error_on:
  591. raise TOMLDecodeError(f"Illegal character {char!r}", src, pos)
  592. pos += 1
  593. def parse_value(
  594. src: str, pos: Pos, parse_float: ParseFloat, nest_lvl: int
  595. ) -> tuple[Pos, Any]:
  596. if nest_lvl > MAX_INLINE_NESTING:
  597. # Pure Python should have raised RecursionError already.
  598. # This ensures mypyc binaries eventually do the same.
  599. raise RecursionError( # pragma: no cover
  600. "TOML inline arrays/tables are nested more than the allowed"
  601. f" {MAX_INLINE_NESTING} levels"
  602. )
  603. try:
  604. char: str | None = src[pos]
  605. except IndexError:
  606. char = None
  607. # IMPORTANT: order conditions based on speed of checking and likelihood
  608. # Basic strings
  609. if char == '"':
  610. if src.startswith('"""', pos):
  611. return parse_multiline_str(src, pos, literal=False)
  612. return parse_one_line_basic_str(src, pos)
  613. # Literal strings
  614. if char == "'":
  615. if src.startswith("'''", pos):
  616. return parse_multiline_str(src, pos, literal=True)
  617. return parse_literal_str(src, pos)
  618. # Booleans
  619. if char == "t":
  620. if src.startswith("true", pos):
  621. return pos + 4, True
  622. if char == "f":
  623. if src.startswith("false", pos):
  624. return pos + 5, False
  625. # Arrays
  626. if char == "[":
  627. return parse_array(src, pos, parse_float, nest_lvl + 1)
  628. # Inline tables
  629. if char == "{":
  630. return parse_inline_table(src, pos, parse_float, nest_lvl + 1)
  631. # Dates and times
  632. datetime_match = RE_DATETIME.match(src, pos)
  633. if datetime_match:
  634. try:
  635. datetime_obj = match_to_datetime(datetime_match)
  636. except ValueError as e:
  637. raise TOMLDecodeError("Invalid date or datetime", src, pos) from e
  638. return datetime_match.end(), datetime_obj
  639. localtime_match = RE_LOCALTIME.match(src, pos)
  640. if localtime_match:
  641. return localtime_match.end(), match_to_localtime(localtime_match)
  642. # Integers and "normal" floats.
  643. # The regex will greedily match any type starting with a decimal
  644. # char, so needs to be located after handling of dates and times.
  645. number_match = RE_NUMBER.match(src, pos)
  646. if number_match:
  647. return number_match.end(), match_to_number(number_match, parse_float)
  648. # Special floats
  649. first_three = src[pos : pos + 3]
  650. if first_three in {"inf", "nan"}:
  651. return pos + 3, parse_float(first_three)
  652. first_four = src[pos : pos + 4]
  653. if first_four in {"-inf", "+inf", "-nan", "+nan"}:
  654. return pos + 4, parse_float(first_four)
  655. raise TOMLDecodeError("Invalid value", src, pos)
  656. def is_unicode_scalar_value(codepoint: int) -> bool:
  657. return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111)
  658. def make_safe_parse_float(parse_float: ParseFloat) -> ParseFloat:
  659. """A decorator to make `parse_float` safe.
  660. `parse_float` must not return dicts or lists, because these types
  661. would be mixed with parsed TOML tables and arrays, thus confusing
  662. the parser. The returned decorated callable raises `ValueError`
  663. instead of returning illegal types.
  664. """
  665. # The default `float` callable never returns illegal types. Optimize it.
  666. if parse_float is float:
  667. return float
  668. def safe_parse_float(float_str: str) -> Any:
  669. float_value = parse_float(float_str)
  670. if isinstance(float_value, (dict, list)):
  671. raise ValueError("parse_float must not return dicts or lists")
  672. return float_value
  673. return safe_parse_float