unittest.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516
  1. # mypy: allow-untyped-defs
  2. """Discover and run std-library "unittest" style tests."""
  3. from __future__ import annotations
  4. from collections.abc import Callable
  5. from collections.abc import Generator
  6. from collections.abc import Iterable
  7. from collections.abc import Iterator
  8. from enum import auto
  9. from enum import Enum
  10. import inspect
  11. import sys
  12. import traceback
  13. import types
  14. from typing import TYPE_CHECKING
  15. from typing import Union
  16. import _pytest._code
  17. from _pytest.compat import is_async_function
  18. from _pytest.config import hookimpl
  19. from _pytest.fixtures import FixtureRequest
  20. from _pytest.monkeypatch import MonkeyPatch
  21. from _pytest.nodes import Collector
  22. from _pytest.nodes import Item
  23. from _pytest.outcomes import exit
  24. from _pytest.outcomes import fail
  25. from _pytest.outcomes import skip
  26. from _pytest.outcomes import xfail
  27. from _pytest.python import Class
  28. from _pytest.python import Function
  29. from _pytest.python import Module
  30. from _pytest.runner import CallInfo
  31. import pytest
  32. if sys.version_info[:2] < (3, 11):
  33. from exceptiongroup import ExceptionGroup
  34. if TYPE_CHECKING:
  35. import unittest
  36. import twisted.trial.unittest
  37. _SysExcInfoType = Union[
  38. tuple[type[BaseException], BaseException, types.TracebackType],
  39. tuple[None, None, None],
  40. ]
  41. def pytest_pycollect_makeitem(
  42. collector: Module | Class, name: str, obj: object
  43. ) -> UnitTestCase | None:
  44. try:
  45. # Has unittest been imported?
  46. ut = sys.modules["unittest"]
  47. # Is obj a subclass of unittest.TestCase?
  48. # Type ignored because `ut` is an opaque module.
  49. if not issubclass(obj, ut.TestCase): # type: ignore
  50. return None
  51. except Exception:
  52. return None
  53. # Is obj a concrete class?
  54. # Abstract classes can't be instantiated so no point collecting them.
  55. if inspect.isabstract(obj):
  56. return None
  57. # Yes, so let's collect it.
  58. return UnitTestCase.from_parent(collector, name=name, obj=obj)
  59. class UnitTestCase(Class):
  60. # Marker for fixturemanger.getfixtureinfo()
  61. # to declare that our children do not support funcargs.
  62. nofuncargs = True
  63. def newinstance(self):
  64. # TestCase __init__ takes the method (test) name. The TestCase
  65. # constructor treats the name "runTest" as a special no-op, so it can be
  66. # used when a dummy instance is needed. While unittest.TestCase has a
  67. # default, some subclasses omit the default (#9610), so always supply
  68. # it.
  69. return self.obj("runTest")
  70. def collect(self) -> Iterable[Item | Collector]:
  71. from unittest import TestLoader
  72. cls = self.obj
  73. if not getattr(cls, "__test__", True):
  74. return
  75. skipped = _is_skipped(cls)
  76. if not skipped:
  77. self._register_unittest_setup_method_fixture(cls)
  78. self._register_unittest_setup_class_fixture(cls)
  79. self._register_setup_class_fixture()
  80. self.session._fixturemanager.parsefactories(self.newinstance(), self.nodeid)
  81. loader = TestLoader()
  82. foundsomething = False
  83. for name in loader.getTestCaseNames(self.obj):
  84. x = getattr(self.obj, name)
  85. if not getattr(x, "__test__", True):
  86. continue
  87. yield TestCaseFunction.from_parent(self, name=name)
  88. foundsomething = True
  89. if not foundsomething:
  90. runtest = getattr(self.obj, "runTest", None)
  91. if runtest is not None:
  92. ut = sys.modules.get("twisted.trial.unittest", None)
  93. if ut is None or runtest != ut.TestCase.runTest:
  94. yield TestCaseFunction.from_parent(self, name="runTest")
  95. def _register_unittest_setup_class_fixture(self, cls: type) -> None:
  96. """Register an auto-use fixture to invoke setUpClass and
  97. tearDownClass (#517)."""
  98. setup = getattr(cls, "setUpClass", None)
  99. teardown = getattr(cls, "tearDownClass", None)
  100. if setup is None and teardown is None:
  101. return None
  102. cleanup = getattr(cls, "doClassCleanups", lambda: None)
  103. def process_teardown_exceptions() -> None:
  104. # tearDown_exceptions is a list set in the class containing exc_infos for errors during
  105. # teardown for the class.
  106. exc_infos = getattr(cls, "tearDown_exceptions", None)
  107. if not exc_infos:
  108. return
  109. exceptions = [exc for (_, exc, _) in exc_infos]
  110. # If a single exception, raise it directly as this provides a more readable
  111. # error (hopefully this will improve in #12255).
  112. if len(exceptions) == 1:
  113. raise exceptions[0]
  114. else:
  115. raise ExceptionGroup("Unittest class cleanup errors", exceptions)
  116. def unittest_setup_class_fixture(
  117. request: FixtureRequest,
  118. ) -> Generator[None]:
  119. cls = request.cls
  120. if _is_skipped(cls):
  121. reason = cls.__unittest_skip_why__
  122. raise pytest.skip.Exception(reason, _use_item_location=True)
  123. if setup is not None:
  124. try:
  125. setup()
  126. # unittest does not call the cleanup function for every BaseException, so we
  127. # follow this here.
  128. except Exception:
  129. cleanup()
  130. process_teardown_exceptions()
  131. raise
  132. yield
  133. try:
  134. if teardown is not None:
  135. teardown()
  136. finally:
  137. cleanup()
  138. process_teardown_exceptions()
  139. self.session._fixturemanager._register_fixture(
  140. # Use a unique name to speed up lookup.
  141. name=f"_unittest_setUpClass_fixture_{cls.__qualname__}",
  142. func=unittest_setup_class_fixture,
  143. nodeid=self.nodeid,
  144. scope="class",
  145. autouse=True,
  146. )
  147. def _register_unittest_setup_method_fixture(self, cls: type) -> None:
  148. """Register an auto-use fixture to invoke setup_method and
  149. teardown_method (#517)."""
  150. setup = getattr(cls, "setup_method", None)
  151. teardown = getattr(cls, "teardown_method", None)
  152. if setup is None and teardown is None:
  153. return None
  154. def unittest_setup_method_fixture(
  155. request: FixtureRequest,
  156. ) -> Generator[None]:
  157. self = request.instance
  158. if _is_skipped(self):
  159. reason = self.__unittest_skip_why__
  160. raise pytest.skip.Exception(reason, _use_item_location=True)
  161. if setup is not None:
  162. setup(self, request.function)
  163. yield
  164. if teardown is not None:
  165. teardown(self, request.function)
  166. self.session._fixturemanager._register_fixture(
  167. # Use a unique name to speed up lookup.
  168. name=f"_unittest_setup_method_fixture_{cls.__qualname__}",
  169. func=unittest_setup_method_fixture,
  170. nodeid=self.nodeid,
  171. scope="function",
  172. autouse=True,
  173. )
  174. class TestCaseFunction(Function):
  175. nofuncargs = True
  176. _excinfo: list[_pytest._code.ExceptionInfo[BaseException]] | None = None
  177. def _getinstance(self):
  178. assert isinstance(self.parent, UnitTestCase)
  179. return self.parent.obj(self.name)
  180. # Backward compat for pytest-django; can be removed after pytest-django
  181. # updates + some slack.
  182. @property
  183. def _testcase(self):
  184. return self.instance
  185. def setup(self) -> None:
  186. # A bound method to be called during teardown() if set (see 'runtest()').
  187. self._explicit_tearDown: Callable[[], None] | None = None
  188. super().setup()
  189. def teardown(self) -> None:
  190. if self._explicit_tearDown is not None:
  191. self._explicit_tearDown()
  192. self._explicit_tearDown = None
  193. self._obj = None
  194. del self._instance
  195. super().teardown()
  196. def startTest(self, testcase: unittest.TestCase) -> None:
  197. pass
  198. def _addexcinfo(self, rawexcinfo: _SysExcInfoType) -> None:
  199. rawexcinfo = _handle_twisted_exc_info(rawexcinfo)
  200. try:
  201. excinfo = _pytest._code.ExceptionInfo[BaseException].from_exc_info(
  202. rawexcinfo # type: ignore[arg-type]
  203. )
  204. # Invoke the attributes to trigger storing the traceback
  205. # trial causes some issue there.
  206. _ = excinfo.value
  207. _ = excinfo.traceback
  208. except TypeError:
  209. try:
  210. try:
  211. values = traceback.format_exception(*rawexcinfo)
  212. values.insert(
  213. 0,
  214. "NOTE: Incompatible Exception Representation, "
  215. "displaying natively:\n\n",
  216. )
  217. fail("".join(values), pytrace=False)
  218. except (fail.Exception, KeyboardInterrupt):
  219. raise
  220. except BaseException:
  221. fail(
  222. "ERROR: Unknown Incompatible Exception "
  223. f"representation:\n{rawexcinfo!r}",
  224. pytrace=False,
  225. )
  226. except KeyboardInterrupt:
  227. raise
  228. except fail.Exception:
  229. excinfo = _pytest._code.ExceptionInfo.from_current()
  230. self.__dict__.setdefault("_excinfo", []).append(excinfo)
  231. def addError(
  232. self, testcase: unittest.TestCase, rawexcinfo: _SysExcInfoType
  233. ) -> None:
  234. try:
  235. if isinstance(rawexcinfo[1], exit.Exception):
  236. exit(rawexcinfo[1].msg)
  237. except TypeError:
  238. pass
  239. self._addexcinfo(rawexcinfo)
  240. def addFailure(
  241. self, testcase: unittest.TestCase, rawexcinfo: _SysExcInfoType
  242. ) -> None:
  243. self._addexcinfo(rawexcinfo)
  244. def addSkip(self, testcase: unittest.TestCase, reason: str) -> None:
  245. try:
  246. raise pytest.skip.Exception(reason, _use_item_location=True)
  247. except skip.Exception:
  248. self._addexcinfo(sys.exc_info())
  249. def addExpectedFailure(
  250. self,
  251. testcase: unittest.TestCase,
  252. rawexcinfo: _SysExcInfoType,
  253. reason: str = "",
  254. ) -> None:
  255. try:
  256. xfail(str(reason))
  257. except xfail.Exception:
  258. self._addexcinfo(sys.exc_info())
  259. def addUnexpectedSuccess(
  260. self,
  261. testcase: unittest.TestCase,
  262. reason: twisted.trial.unittest.Todo | None = None,
  263. ) -> None:
  264. msg = "Unexpected success"
  265. if reason:
  266. msg += f": {reason.reason}"
  267. # Preserve unittest behaviour - fail the test. Explicitly not an XPASS.
  268. try:
  269. fail(msg, pytrace=False)
  270. except fail.Exception:
  271. self._addexcinfo(sys.exc_info())
  272. def addSuccess(self, testcase: unittest.TestCase) -> None:
  273. pass
  274. def stopTest(self, testcase: unittest.TestCase) -> None:
  275. pass
  276. def addDuration(self, testcase: unittest.TestCase, elapsed: float) -> None:
  277. pass
  278. def runtest(self) -> None:
  279. from _pytest.debugging import maybe_wrap_pytest_function_for_tracing
  280. testcase = self.instance
  281. assert testcase is not None
  282. maybe_wrap_pytest_function_for_tracing(self)
  283. # Let the unittest framework handle async functions.
  284. if is_async_function(self.obj):
  285. testcase(result=self)
  286. else:
  287. # When --pdb is given, we want to postpone calling tearDown() otherwise
  288. # when entering the pdb prompt, tearDown() would have probably cleaned up
  289. # instance variables, which makes it difficult to debug.
  290. # Arguably we could always postpone tearDown(), but this changes the moment where the
  291. # TestCase instance interacts with the results object, so better to only do it
  292. # when absolutely needed.
  293. # We need to consider if the test itself is skipped, or the whole class.
  294. assert isinstance(self.parent, UnitTestCase)
  295. skipped = _is_skipped(self.obj) or _is_skipped(self.parent.obj)
  296. if self.config.getoption("usepdb") and not skipped:
  297. self._explicit_tearDown = testcase.tearDown
  298. setattr(testcase, "tearDown", lambda *args: None)
  299. # We need to update the actual bound method with self.obj, because
  300. # wrap_pytest_function_for_tracing replaces self.obj by a wrapper.
  301. setattr(testcase, self.name, self.obj)
  302. try:
  303. testcase(result=self)
  304. finally:
  305. delattr(testcase, self.name)
  306. def _traceback_filter(
  307. self, excinfo: _pytest._code.ExceptionInfo[BaseException]
  308. ) -> _pytest._code.Traceback:
  309. traceback = super()._traceback_filter(excinfo)
  310. ntraceback = traceback.filter(
  311. lambda x: not x.frame.f_globals.get("__unittest"),
  312. )
  313. if not ntraceback:
  314. ntraceback = traceback
  315. return ntraceback
  316. @hookimpl(tryfirst=True)
  317. def pytest_runtest_makereport(item: Item, call: CallInfo[None]) -> None:
  318. if isinstance(item, TestCaseFunction):
  319. if item._excinfo:
  320. call.excinfo = item._excinfo.pop(0)
  321. try:
  322. del call.result
  323. except AttributeError:
  324. pass
  325. # Convert unittest.SkipTest to pytest.skip.
  326. # This is actually only needed for nose, which reuses unittest.SkipTest for
  327. # its own nose.SkipTest. For unittest TestCases, SkipTest is already
  328. # handled internally, and doesn't reach here.
  329. unittest = sys.modules.get("unittest")
  330. if unittest and call.excinfo and isinstance(call.excinfo.value, unittest.SkipTest):
  331. excinfo = call.excinfo
  332. call2 = CallInfo[None].from_call(
  333. lambda: pytest.skip(str(excinfo.value)), call.when
  334. )
  335. call.excinfo = call2.excinfo
  336. def _is_skipped(obj) -> bool:
  337. """Return True if the given object has been marked with @unittest.skip."""
  338. return bool(getattr(obj, "__unittest_skip__", False))
  339. def pytest_configure() -> None:
  340. """Register the TestCaseFunction class as an IReporter if twisted.trial is available."""
  341. if _get_twisted_version() is not TwistedVersion.NotInstalled:
  342. from twisted.trial.itrial import IReporter
  343. from zope.interface import classImplements
  344. classImplements(TestCaseFunction, IReporter)
  345. class TwistedVersion(Enum):
  346. """
  347. The Twisted version installed in the environment.
  348. We have different workarounds in place for different versions of Twisted.
  349. """
  350. # Twisted version 24 or prior.
  351. Version24 = auto()
  352. # Twisted version 25 or later.
  353. Version25 = auto()
  354. # Twisted version is not available.
  355. NotInstalled = auto()
  356. def _get_twisted_version() -> TwistedVersion:
  357. # We need to check if "twisted.trial.unittest" is specifically present in sys.modules.
  358. # This is because we intend to integrate with Trial only when it's actively running
  359. # the test suite, but not needed when only other Twisted components are in use.
  360. if "twisted.trial.unittest" not in sys.modules:
  361. return TwistedVersion.NotInstalled
  362. import importlib.metadata
  363. import packaging.version
  364. version_str = importlib.metadata.version("twisted")
  365. version = packaging.version.parse(version_str)
  366. if version.major <= 24:
  367. return TwistedVersion.Version24
  368. else:
  369. return TwistedVersion.Version25
  370. # Name of the attribute in `twisted.python.Failure` instances that stores
  371. # the `sys.exc_info()` tuple.
  372. # See twisted.trial support in `pytest_runtest_protocol`.
  373. TWISTED_RAW_EXCINFO_ATTR = "_twisted_raw_excinfo"
  374. @hookimpl(wrapper=True)
  375. def pytest_runtest_protocol(item: Item) -> Iterator[None]:
  376. if _get_twisted_version() is TwistedVersion.Version24:
  377. import twisted.python.failure as ut
  378. # Monkeypatch `Failure.__init__` to store the raw exception info.
  379. original__init__ = ut.Failure.__init__
  380. def store_raw_exception_info(
  381. self, exc_value=None, exc_type=None, exc_tb=None, captureVars=None
  382. ): # pragma: no cover
  383. if exc_value is None:
  384. raw_exc_info = sys.exc_info()
  385. else:
  386. if exc_type is None:
  387. exc_type = type(exc_value)
  388. if exc_tb is None:
  389. exc_tb = sys.exc_info()[2]
  390. raw_exc_info = (exc_type, exc_value, exc_tb)
  391. setattr(self, TWISTED_RAW_EXCINFO_ATTR, tuple(raw_exc_info))
  392. try:
  393. original__init__(
  394. self, exc_value, exc_type, exc_tb, captureVars=captureVars
  395. )
  396. except TypeError: # pragma: no cover
  397. original__init__(self, exc_value, exc_type, exc_tb)
  398. with MonkeyPatch.context() as patcher:
  399. patcher.setattr(ut.Failure, "__init__", store_raw_exception_info)
  400. return (yield)
  401. else:
  402. return (yield)
  403. def _handle_twisted_exc_info(
  404. rawexcinfo: _SysExcInfoType | BaseException,
  405. ) -> _SysExcInfoType:
  406. """
  407. Twisted passes a custom Failure instance to `addError()` instead of using `sys.exc_info()`.
  408. Therefore, if `rawexcinfo` is a `Failure` instance, convert it into the equivalent `sys.exc_info()` tuple
  409. as expected by pytest.
  410. """
  411. twisted_version = _get_twisted_version()
  412. if twisted_version is TwistedVersion.NotInstalled:
  413. # Unfortunately, because we cannot import `twisted.python.failure` at the top of the file
  414. # and use it in the signature, we need to use `type:ignore` here because we cannot narrow
  415. # the type properly in the `if` statement above.
  416. return rawexcinfo # type:ignore[return-value]
  417. elif twisted_version is TwistedVersion.Version24:
  418. # Twisted calls addError() passing its own classes (like `twisted.python.Failure`), which violates
  419. # the `addError()` signature, so we extract the original `sys.exc_info()` tuple which is stored
  420. # in the object.
  421. if hasattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR):
  422. saved_exc_info = getattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR)
  423. # Delete the attribute from the original object to avoid leaks.
  424. delattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR)
  425. return saved_exc_info # type:ignore[no-any-return]
  426. return rawexcinfo # type:ignore[return-value]
  427. elif twisted_version is TwistedVersion.Version25:
  428. if isinstance(rawexcinfo, BaseException):
  429. import twisted.python.failure
  430. if isinstance(rawexcinfo, twisted.python.failure.Failure):
  431. tb = rawexcinfo.__traceback__
  432. if tb is None:
  433. tb = sys.exc_info()[2]
  434. return type(rawexcinfo.value), rawexcinfo.value, tb
  435. return rawexcinfo # type:ignore[return-value]
  436. else:
  437. # Ideally we would use assert_never() here, but it is not available in all Python versions
  438. # we support, plus we do not require `type_extensions` currently.
  439. assert False, f"Unexpected Twisted version: {twisted_version}"