argparsing.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535
  1. # mypy: allow-untyped-defs
  2. from __future__ import annotations
  3. import argparse
  4. from collections.abc import Callable
  5. from collections.abc import Mapping
  6. from collections.abc import Sequence
  7. import os
  8. from typing import Any
  9. from typing import cast
  10. from typing import final
  11. from typing import Literal
  12. from typing import NoReturn
  13. import _pytest._io
  14. from _pytest.config.exceptions import UsageError
  15. from _pytest.deprecated import check_ispytest
  16. FILE_OR_DIR = "file_or_dir"
  17. class NotSet:
  18. def __repr__(self) -> str:
  19. return "<notset>"
  20. NOT_SET = NotSet()
  21. @final
  22. class Parser:
  23. """Parser for command line arguments and ini-file values.
  24. :ivar extra_info: Dict of generic param -> value to display in case
  25. there's an error processing the command line arguments.
  26. """
  27. prog: str | None = None
  28. def __init__(
  29. self,
  30. usage: str | None = None,
  31. processopt: Callable[[Argument], None] | None = None,
  32. *,
  33. _ispytest: bool = False,
  34. ) -> None:
  35. check_ispytest(_ispytest)
  36. self._anonymous = OptionGroup("Custom options", parser=self, _ispytest=True)
  37. self._groups: list[OptionGroup] = []
  38. self._processopt = processopt
  39. self._usage = usage
  40. self._inidict: dict[str, tuple[str, str | None, Any]] = {}
  41. self._ininames: list[str] = []
  42. self.extra_info: dict[str, Any] = {}
  43. def processoption(self, option: Argument) -> None:
  44. if self._processopt:
  45. if option.dest:
  46. self._processopt(option)
  47. def getgroup(
  48. self, name: str, description: str = "", after: str | None = None
  49. ) -> OptionGroup:
  50. """Get (or create) a named option Group.
  51. :param name: Name of the option group.
  52. :param description: Long description for --help output.
  53. :param after: Name of another group, used for ordering --help output.
  54. :returns: The option group.
  55. The returned group object has an ``addoption`` method with the same
  56. signature as :func:`parser.addoption <pytest.Parser.addoption>` but
  57. will be shown in the respective group in the output of
  58. ``pytest --help``.
  59. """
  60. for group in self._groups:
  61. if group.name == name:
  62. return group
  63. group = OptionGroup(name, description, parser=self, _ispytest=True)
  64. i = 0
  65. for i, grp in enumerate(self._groups):
  66. if grp.name == after:
  67. break
  68. self._groups.insert(i + 1, group)
  69. return group
  70. def addoption(self, *opts: str, **attrs: Any) -> None:
  71. """Register a command line option.
  72. :param opts:
  73. Option names, can be short or long options.
  74. :param attrs:
  75. Same attributes as the argparse library's :meth:`add_argument()
  76. <argparse.ArgumentParser.add_argument>` function accepts.
  77. After command line parsing, options are available on the pytest config
  78. object via ``config.option.NAME`` where ``NAME`` is usually set
  79. by passing a ``dest`` attribute, for example
  80. ``addoption("--long", dest="NAME", ...)``.
  81. """
  82. self._anonymous.addoption(*opts, **attrs)
  83. def parse(
  84. self,
  85. args: Sequence[str | os.PathLike[str]],
  86. namespace: argparse.Namespace | None = None,
  87. ) -> argparse.Namespace:
  88. from _pytest._argcomplete import try_argcomplete
  89. self.optparser = self._getparser()
  90. try_argcomplete(self.optparser)
  91. strargs = [os.fspath(x) for x in args]
  92. return self.optparser.parse_args(strargs, namespace=namespace)
  93. def _getparser(self) -> MyOptionParser:
  94. from _pytest._argcomplete import filescompleter
  95. optparser = MyOptionParser(self, self.extra_info, prog=self.prog)
  96. groups = [*self._groups, self._anonymous]
  97. for group in groups:
  98. if group.options:
  99. desc = group.description or group.name
  100. arggroup = optparser.add_argument_group(desc)
  101. for option in group.options:
  102. n = option.names()
  103. a = option.attrs()
  104. arggroup.add_argument(*n, **a)
  105. file_or_dir_arg = optparser.add_argument(FILE_OR_DIR, nargs="*")
  106. # bash like autocompletion for dirs (appending '/')
  107. # Type ignored because typeshed doesn't know about argcomplete.
  108. file_or_dir_arg.completer = filescompleter # type: ignore
  109. return optparser
  110. def parse_setoption(
  111. self,
  112. args: Sequence[str | os.PathLike[str]],
  113. option: argparse.Namespace,
  114. namespace: argparse.Namespace | None = None,
  115. ) -> list[str]:
  116. parsedoption = self.parse(args, namespace=namespace)
  117. for name, value in parsedoption.__dict__.items():
  118. setattr(option, name, value)
  119. return cast(list[str], getattr(parsedoption, FILE_OR_DIR))
  120. def parse_known_args(
  121. self,
  122. args: Sequence[str | os.PathLike[str]],
  123. namespace: argparse.Namespace | None = None,
  124. ) -> argparse.Namespace:
  125. """Parse the known arguments at this point.
  126. :returns: An argparse namespace object.
  127. """
  128. return self.parse_known_and_unknown_args(args, namespace=namespace)[0]
  129. def parse_known_and_unknown_args(
  130. self,
  131. args: Sequence[str | os.PathLike[str]],
  132. namespace: argparse.Namespace | None = None,
  133. ) -> tuple[argparse.Namespace, list[str]]:
  134. """Parse the known arguments at this point, and also return the
  135. remaining unknown arguments.
  136. :returns:
  137. A tuple containing an argparse namespace object for the known
  138. arguments, and a list of the unknown arguments.
  139. """
  140. optparser = self._getparser()
  141. strargs = [os.fspath(x) for x in args]
  142. return optparser.parse_known_args(strargs, namespace=namespace)
  143. def addini(
  144. self,
  145. name: str,
  146. help: str,
  147. type: Literal[
  148. "string", "paths", "pathlist", "args", "linelist", "bool", "int", "float"
  149. ]
  150. | None = None,
  151. default: Any = NOT_SET,
  152. ) -> None:
  153. """Register an ini-file option.
  154. :param name:
  155. Name of the ini-variable.
  156. :param type:
  157. Type of the variable. Can be:
  158. * ``string``: a string
  159. * ``bool``: a boolean
  160. * ``args``: a list of strings, separated as in a shell
  161. * ``linelist``: a list of strings, separated by line breaks
  162. * ``paths``: a list of :class:`pathlib.Path`, separated as in a shell
  163. * ``pathlist``: a list of ``py.path``, separated as in a shell
  164. * ``int``: an integer
  165. * ``float``: a floating-point number
  166. .. versionadded:: 8.4
  167. The ``float`` and ``int`` types.
  168. For ``paths`` and ``pathlist`` types, they are considered relative to the ini-file.
  169. In case the execution is happening without an ini-file defined,
  170. they will be considered relative to the current working directory (for example with ``--override-ini``).
  171. .. versionadded:: 7.0
  172. The ``paths`` variable type.
  173. .. versionadded:: 8.1
  174. Use the current working directory to resolve ``paths`` and ``pathlist`` in the absence of an ini-file.
  175. Defaults to ``string`` if ``None`` or not passed.
  176. :param default:
  177. Default value if no ini-file option exists but is queried.
  178. The value of ini-variables can be retrieved via a call to
  179. :py:func:`config.getini(name) <pytest.Config.getini>`.
  180. """
  181. assert type in (
  182. None,
  183. "string",
  184. "paths",
  185. "pathlist",
  186. "args",
  187. "linelist",
  188. "bool",
  189. "int",
  190. "float",
  191. )
  192. if default is NOT_SET:
  193. default = get_ini_default_for_type(type)
  194. self._inidict[name] = (help, type, default)
  195. self._ininames.append(name)
  196. def get_ini_default_for_type(
  197. type: Literal[
  198. "string", "paths", "pathlist", "args", "linelist", "bool", "int", "float"
  199. ]
  200. | None,
  201. ) -> Any:
  202. """
  203. Used by addini to get the default value for a given ini-option type, when
  204. default is not supplied.
  205. """
  206. if type is None:
  207. return ""
  208. elif type in ("paths", "pathlist", "args", "linelist"):
  209. return []
  210. elif type == "bool":
  211. return False
  212. elif type == "int":
  213. return 0
  214. elif type == "float":
  215. return 0.0
  216. else:
  217. return ""
  218. class ArgumentError(Exception):
  219. """Raised if an Argument instance is created with invalid or
  220. inconsistent arguments."""
  221. def __init__(self, msg: str, option: Argument | str) -> None:
  222. self.msg = msg
  223. self.option_id = str(option)
  224. def __str__(self) -> str:
  225. if self.option_id:
  226. return f"option {self.option_id}: {self.msg}"
  227. else:
  228. return self.msg
  229. class Argument:
  230. """Class that mimics the necessary behaviour of optparse.Option.
  231. It's currently a least effort implementation and ignoring choices
  232. and integer prefixes.
  233. https://docs.python.org/3/library/optparse.html#optparse-standard-option-types
  234. """
  235. def __init__(self, *names: str, **attrs: Any) -> None:
  236. """Store params in private vars for use in add_argument."""
  237. self._attrs = attrs
  238. self._short_opts: list[str] = []
  239. self._long_opts: list[str] = []
  240. try:
  241. self.type = attrs["type"]
  242. except KeyError:
  243. pass
  244. try:
  245. # Attribute existence is tested in Config._processopt.
  246. self.default = attrs["default"]
  247. except KeyError:
  248. pass
  249. self._set_opt_strings(names)
  250. dest: str | None = attrs.get("dest")
  251. if dest:
  252. self.dest = dest
  253. elif self._long_opts:
  254. self.dest = self._long_opts[0][2:].replace("-", "_")
  255. else:
  256. try:
  257. self.dest = self._short_opts[0][1:]
  258. except IndexError as e:
  259. self.dest = "???" # Needed for the error repr.
  260. raise ArgumentError("need a long or short option", self) from e
  261. def names(self) -> list[str]:
  262. return self._short_opts + self._long_opts
  263. def attrs(self) -> Mapping[str, Any]:
  264. # Update any attributes set by processopt.
  265. attrs = "default dest help".split()
  266. attrs.append(self.dest)
  267. for attr in attrs:
  268. try:
  269. self._attrs[attr] = getattr(self, attr)
  270. except AttributeError:
  271. pass
  272. return self._attrs
  273. def _set_opt_strings(self, opts: Sequence[str]) -> None:
  274. """Directly from optparse.
  275. Might not be necessary as this is passed to argparse later on.
  276. """
  277. for opt in opts:
  278. if len(opt) < 2:
  279. raise ArgumentError(
  280. f"invalid option string {opt!r}: "
  281. "must be at least two characters long",
  282. self,
  283. )
  284. elif len(opt) == 2:
  285. if not (opt[0] == "-" and opt[1] != "-"):
  286. raise ArgumentError(
  287. f"invalid short option string {opt!r}: "
  288. "must be of the form -x, (x any non-dash char)",
  289. self,
  290. )
  291. self._short_opts.append(opt)
  292. else:
  293. if not (opt[0:2] == "--" and opt[2] != "-"):
  294. raise ArgumentError(
  295. f"invalid long option string {opt!r}: "
  296. "must start with --, followed by non-dash",
  297. self,
  298. )
  299. self._long_opts.append(opt)
  300. def __repr__(self) -> str:
  301. args: list[str] = []
  302. if self._short_opts:
  303. args += ["_short_opts: " + repr(self._short_opts)]
  304. if self._long_opts:
  305. args += ["_long_opts: " + repr(self._long_opts)]
  306. args += ["dest: " + repr(self.dest)]
  307. if hasattr(self, "type"):
  308. args += ["type: " + repr(self.type)]
  309. if hasattr(self, "default"):
  310. args += ["default: " + repr(self.default)]
  311. return "Argument({})".format(", ".join(args))
  312. class OptionGroup:
  313. """A group of options shown in its own section."""
  314. def __init__(
  315. self,
  316. name: str,
  317. description: str = "",
  318. parser: Parser | None = None,
  319. *,
  320. _ispytest: bool = False,
  321. ) -> None:
  322. check_ispytest(_ispytest)
  323. self.name = name
  324. self.description = description
  325. self.options: list[Argument] = []
  326. self.parser = parser
  327. def addoption(self, *opts: str, **attrs: Any) -> None:
  328. """Add an option to this group.
  329. If a shortened version of a long option is specified, it will
  330. be suppressed in the help. ``addoption('--twowords', '--two-words')``
  331. results in help showing ``--two-words`` only, but ``--twowords`` gets
  332. accepted **and** the automatic destination is in ``args.twowords``.
  333. :param opts:
  334. Option names, can be short or long options.
  335. :param attrs:
  336. Same attributes as the argparse library's :meth:`add_argument()
  337. <argparse.ArgumentParser.add_argument>` function accepts.
  338. """
  339. conflict = set(opts).intersection(
  340. name for opt in self.options for name in opt.names()
  341. )
  342. if conflict:
  343. raise ValueError(f"option names {conflict} already added")
  344. option = Argument(*opts, **attrs)
  345. self._addoption_instance(option, shortupper=False)
  346. def _addoption(self, *opts: str, **attrs: Any) -> None:
  347. option = Argument(*opts, **attrs)
  348. self._addoption_instance(option, shortupper=True)
  349. def _addoption_instance(self, option: Argument, shortupper: bool = False) -> None:
  350. if not shortupper:
  351. for opt in option._short_opts:
  352. if opt[0] == "-" and opt[1].islower():
  353. raise ValueError("lowercase shortoptions reserved")
  354. if self.parser:
  355. self.parser.processoption(option)
  356. self.options.append(option)
  357. class MyOptionParser(argparse.ArgumentParser):
  358. def __init__(
  359. self,
  360. parser: Parser,
  361. extra_info: dict[str, Any] | None = None,
  362. prog: str | None = None,
  363. ) -> None:
  364. self._parser = parser
  365. super().__init__(
  366. prog=prog,
  367. usage=parser._usage,
  368. add_help=False,
  369. formatter_class=DropShorterLongHelpFormatter,
  370. allow_abbrev=False,
  371. fromfile_prefix_chars="@",
  372. )
  373. # extra_info is a dict of (param -> value) to display if there's
  374. # an usage error to provide more contextual information to the user.
  375. self.extra_info = extra_info if extra_info else {}
  376. def error(self, message: str) -> NoReturn:
  377. """Transform argparse error message into UsageError."""
  378. msg = f"{self.prog}: error: {message}"
  379. if hasattr(self._parser, "_config_source_hint"):
  380. msg = f"{msg} ({self._parser._config_source_hint})"
  381. raise UsageError(self.format_usage() + msg)
  382. # Type ignored because typeshed has a very complex type in the superclass.
  383. def parse_args( # type: ignore
  384. self,
  385. args: Sequence[str] | None = None,
  386. namespace: argparse.Namespace | None = None,
  387. ) -> argparse.Namespace:
  388. """Allow splitting of positional arguments."""
  389. parsed, unrecognized = self.parse_known_args(args, namespace)
  390. if unrecognized:
  391. for arg in unrecognized:
  392. if arg and arg[0] == "-":
  393. lines = [
  394. "unrecognized arguments: {}".format(" ".join(unrecognized))
  395. ]
  396. for k, v in sorted(self.extra_info.items()):
  397. lines.append(f" {k}: {v}")
  398. self.error("\n".join(lines))
  399. getattr(parsed, FILE_OR_DIR).extend(unrecognized)
  400. return parsed
  401. class DropShorterLongHelpFormatter(argparse.HelpFormatter):
  402. """Shorten help for long options that differ only in extra hyphens.
  403. - Collapse **long** options that are the same except for extra hyphens.
  404. - Shortcut if there are only two options and one of them is a short one.
  405. - Cache result on the action object as this is called at least 2 times.
  406. """
  407. def __init__(self, *args: Any, **kwargs: Any) -> None:
  408. # Use more accurate terminal width.
  409. if "width" not in kwargs:
  410. kwargs["width"] = _pytest._io.get_terminal_width()
  411. super().__init__(*args, **kwargs)
  412. def _format_action_invocation(self, action: argparse.Action) -> str:
  413. orgstr = super()._format_action_invocation(action)
  414. if orgstr and orgstr[0] != "-": # only optional arguments
  415. return orgstr
  416. res: str | None = getattr(action, "_formatted_action_invocation", None)
  417. if res:
  418. return res
  419. options = orgstr.split(", ")
  420. if len(options) == 2 and (len(options[0]) == 2 or len(options[1]) == 2):
  421. # a shortcut for '-h, --help' or '--abc', '-a'
  422. action._formatted_action_invocation = orgstr # type: ignore
  423. return orgstr
  424. return_list = []
  425. short_long: dict[str, str] = {}
  426. for option in options:
  427. if len(option) == 2 or option[2] == " ":
  428. continue
  429. if not option.startswith("--"):
  430. raise ArgumentError(
  431. f'long optional argument without "--": [{option}]', option
  432. )
  433. xxoption = option[2:]
  434. shortened = xxoption.replace("-", "")
  435. if shortened not in short_long or len(short_long[shortened]) < len(
  436. xxoption
  437. ):
  438. short_long[shortened] = xxoption
  439. # now short_long has been filled out to the longest with dashes
  440. # **and** we keep the right option ordering from add_argument
  441. for option in options:
  442. if len(option) == 2 or option[2] == " ":
  443. return_list.append(option)
  444. if option[2:] == short_long.get(option.replace("-", "")):
  445. return_list.append(option.replace(" ", "=", 1))
  446. formatted_action_invocation = ", ".join(return_list)
  447. action._formatted_action_invocation = formatted_action_invocation # type: ignore
  448. return formatted_action_invocation
  449. def _split_lines(self, text, width):
  450. """Wrap lines after splitting on original newlines.
  451. This allows to have explicit line breaks in the help text.
  452. """
  453. import textwrap
  454. lines = []
  455. for line in text.splitlines():
  456. lines.extend(textwrap.wrap(line.strip(), width))
  457. return lines