default.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383
  1. """
  2. Custom transports, with nicely configured defaults.
  3. The following additional keyword arguments are currently supported by httpcore...
  4. * uds: str
  5. * local_address: str
  6. * retries: int
  7. Example usages...
  8. # Disable HTTP/2 on a single specific domain.
  9. mounts = {
  10. "all://": httpx.HTTPTransport(http2=True),
  11. "all://*example.org": httpx.HTTPTransport()
  12. }
  13. # Using advanced httpcore configuration, with connection retries.
  14. transport = httpx.HTTPTransport(retries=1)
  15. client = httpx.Client(transport=transport)
  16. # Using advanced httpcore configuration, with unix domain sockets.
  17. transport = httpx.HTTPTransport(uds="socket.uds")
  18. client = httpx.Client(transport=transport)
  19. """
  20. import contextlib
  21. import typing
  22. from types import TracebackType
  23. import httpcore
  24. from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context
  25. from .._exceptions import (
  26. ConnectError,
  27. ConnectTimeout,
  28. LocalProtocolError,
  29. NetworkError,
  30. PoolTimeout,
  31. ProtocolError,
  32. ProxyError,
  33. ReadError,
  34. ReadTimeout,
  35. RemoteProtocolError,
  36. TimeoutException,
  37. UnsupportedProtocol,
  38. WriteError,
  39. WriteTimeout,
  40. )
  41. from .._models import Request, Response
  42. from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream, VerifyTypes
  43. from .._urls import URL
  44. from .base import AsyncBaseTransport, BaseTransport
  45. T = typing.TypeVar("T", bound="HTTPTransport")
  46. A = typing.TypeVar("A", bound="AsyncHTTPTransport")
  47. SOCKET_OPTION = typing.Union[
  48. typing.Tuple[int, int, int],
  49. typing.Tuple[int, int, typing.Union[bytes, bytearray]],
  50. typing.Tuple[int, int, None, int],
  51. ]
  52. @contextlib.contextmanager
  53. def map_httpcore_exceptions() -> typing.Iterator[None]:
  54. try:
  55. yield
  56. except Exception as exc:
  57. mapped_exc = None
  58. for from_exc, to_exc in HTTPCORE_EXC_MAP.items():
  59. if not isinstance(exc, from_exc):
  60. continue
  61. # We want to map to the most specific exception we can find.
  62. # Eg if `exc` is an `httpcore.ReadTimeout`, we want to map to
  63. # `httpx.ReadTimeout`, not just `httpx.TimeoutException`.
  64. if mapped_exc is None or issubclass(to_exc, mapped_exc):
  65. mapped_exc = to_exc
  66. if mapped_exc is None: # pragma: no cover
  67. raise
  68. message = str(exc)
  69. raise mapped_exc(message) from exc
  70. HTTPCORE_EXC_MAP = {
  71. httpcore.TimeoutException: TimeoutException,
  72. httpcore.ConnectTimeout: ConnectTimeout,
  73. httpcore.ReadTimeout: ReadTimeout,
  74. httpcore.WriteTimeout: WriteTimeout,
  75. httpcore.PoolTimeout: PoolTimeout,
  76. httpcore.NetworkError: NetworkError,
  77. httpcore.ConnectError: ConnectError,
  78. httpcore.ReadError: ReadError,
  79. httpcore.WriteError: WriteError,
  80. httpcore.ProxyError: ProxyError,
  81. httpcore.UnsupportedProtocol: UnsupportedProtocol,
  82. httpcore.ProtocolError: ProtocolError,
  83. httpcore.LocalProtocolError: LocalProtocolError,
  84. httpcore.RemoteProtocolError: RemoteProtocolError,
  85. }
  86. class ResponseStream(SyncByteStream):
  87. def __init__(self, httpcore_stream: typing.Iterable[bytes]) -> None:
  88. self._httpcore_stream = httpcore_stream
  89. def __iter__(self) -> typing.Iterator[bytes]:
  90. with map_httpcore_exceptions():
  91. for part in self._httpcore_stream:
  92. yield part
  93. def close(self) -> None:
  94. if hasattr(self._httpcore_stream, "close"):
  95. self._httpcore_stream.close()
  96. class HTTPTransport(BaseTransport):
  97. def __init__(
  98. self,
  99. verify: VerifyTypes = True,
  100. cert: typing.Optional[CertTypes] = None,
  101. http1: bool = True,
  102. http2: bool = False,
  103. limits: Limits = DEFAULT_LIMITS,
  104. trust_env: bool = True,
  105. proxy: typing.Optional[ProxyTypes] = None,
  106. uds: typing.Optional[str] = None,
  107. local_address: typing.Optional[str] = None,
  108. retries: int = 0,
  109. socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None,
  110. ) -> None:
  111. ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env)
  112. proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy
  113. if proxy is None:
  114. self._pool = httpcore.ConnectionPool(
  115. ssl_context=ssl_context,
  116. max_connections=limits.max_connections,
  117. max_keepalive_connections=limits.max_keepalive_connections,
  118. keepalive_expiry=limits.keepalive_expiry,
  119. http1=http1,
  120. http2=http2,
  121. uds=uds,
  122. local_address=local_address,
  123. retries=retries,
  124. socket_options=socket_options,
  125. )
  126. elif proxy.url.scheme in ("http", "https"):
  127. self._pool = httpcore.HTTPProxy(
  128. proxy_url=httpcore.URL(
  129. scheme=proxy.url.raw_scheme,
  130. host=proxy.url.raw_host,
  131. port=proxy.url.port,
  132. target=proxy.url.raw_path,
  133. ),
  134. proxy_auth=proxy.raw_auth,
  135. proxy_headers=proxy.headers.raw,
  136. ssl_context=ssl_context,
  137. proxy_ssl_context=proxy.ssl_context,
  138. max_connections=limits.max_connections,
  139. max_keepalive_connections=limits.max_keepalive_connections,
  140. keepalive_expiry=limits.keepalive_expiry,
  141. http1=http1,
  142. http2=http2,
  143. socket_options=socket_options,
  144. )
  145. elif proxy.url.scheme == "socks5":
  146. try:
  147. import socksio # noqa
  148. except ImportError: # pragma: no cover
  149. raise ImportError(
  150. "Using SOCKS proxy, but the 'socksio' package is not installed. "
  151. "Make sure to install httpx using `pip install httpx[socks]`."
  152. ) from None
  153. self._pool = httpcore.SOCKSProxy(
  154. proxy_url=httpcore.URL(
  155. scheme=proxy.url.raw_scheme,
  156. host=proxy.url.raw_host,
  157. port=proxy.url.port,
  158. target=proxy.url.raw_path,
  159. ),
  160. proxy_auth=proxy.raw_auth,
  161. ssl_context=ssl_context,
  162. max_connections=limits.max_connections,
  163. max_keepalive_connections=limits.max_keepalive_connections,
  164. keepalive_expiry=limits.keepalive_expiry,
  165. http1=http1,
  166. http2=http2,
  167. )
  168. else: # pragma: no cover
  169. raise ValueError(
  170. "Proxy protocol must be either 'http', 'https', or 'socks5',"
  171. f" but got {proxy.url.scheme!r}."
  172. )
  173. def __enter__(self: T) -> T: # Use generics for subclass support.
  174. self._pool.__enter__()
  175. return self
  176. def __exit__(
  177. self,
  178. exc_type: typing.Optional[typing.Type[BaseException]] = None,
  179. exc_value: typing.Optional[BaseException] = None,
  180. traceback: typing.Optional[TracebackType] = None,
  181. ) -> None:
  182. with map_httpcore_exceptions():
  183. self._pool.__exit__(exc_type, exc_value, traceback)
  184. def handle_request(
  185. self,
  186. request: Request,
  187. ) -> Response:
  188. assert isinstance(request.stream, SyncByteStream)
  189. req = httpcore.Request(
  190. method=request.method,
  191. url=httpcore.URL(
  192. scheme=request.url.raw_scheme,
  193. host=request.url.raw_host,
  194. port=request.url.port,
  195. target=request.url.raw_path,
  196. ),
  197. headers=request.headers.raw,
  198. content=request.stream,
  199. extensions=request.extensions,
  200. )
  201. with map_httpcore_exceptions():
  202. resp = self._pool.handle_request(req)
  203. assert isinstance(resp.stream, typing.Iterable)
  204. return Response(
  205. status_code=resp.status,
  206. headers=resp.headers,
  207. stream=ResponseStream(resp.stream),
  208. extensions=resp.extensions,
  209. )
  210. def close(self) -> None:
  211. self._pool.close()
  212. class AsyncResponseStream(AsyncByteStream):
  213. def __init__(self, httpcore_stream: typing.AsyncIterable[bytes]) -> None:
  214. self._httpcore_stream = httpcore_stream
  215. async def __aiter__(self) -> typing.AsyncIterator[bytes]:
  216. with map_httpcore_exceptions():
  217. async for part in self._httpcore_stream:
  218. yield part
  219. async def aclose(self) -> None:
  220. if hasattr(self._httpcore_stream, "aclose"):
  221. await self._httpcore_stream.aclose()
  222. class AsyncHTTPTransport(AsyncBaseTransport):
  223. def __init__(
  224. self,
  225. verify: VerifyTypes = True,
  226. cert: typing.Optional[CertTypes] = None,
  227. http1: bool = True,
  228. http2: bool = False,
  229. limits: Limits = DEFAULT_LIMITS,
  230. trust_env: bool = True,
  231. proxy: typing.Optional[ProxyTypes] = None,
  232. uds: typing.Optional[str] = None,
  233. local_address: typing.Optional[str] = None,
  234. retries: int = 0,
  235. socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None,
  236. ) -> None:
  237. ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env)
  238. proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy
  239. if proxy is None:
  240. self._pool = httpcore.AsyncConnectionPool(
  241. ssl_context=ssl_context,
  242. max_connections=limits.max_connections,
  243. max_keepalive_connections=limits.max_keepalive_connections,
  244. keepalive_expiry=limits.keepalive_expiry,
  245. http1=http1,
  246. http2=http2,
  247. uds=uds,
  248. local_address=local_address,
  249. retries=retries,
  250. socket_options=socket_options,
  251. )
  252. elif proxy.url.scheme in ("http", "https"):
  253. self._pool = httpcore.AsyncHTTPProxy(
  254. proxy_url=httpcore.URL(
  255. scheme=proxy.url.raw_scheme,
  256. host=proxy.url.raw_host,
  257. port=proxy.url.port,
  258. target=proxy.url.raw_path,
  259. ),
  260. proxy_auth=proxy.raw_auth,
  261. proxy_headers=proxy.headers.raw,
  262. ssl_context=ssl_context,
  263. max_connections=limits.max_connections,
  264. max_keepalive_connections=limits.max_keepalive_connections,
  265. keepalive_expiry=limits.keepalive_expiry,
  266. http1=http1,
  267. http2=http2,
  268. socket_options=socket_options,
  269. )
  270. elif proxy.url.scheme == "socks5":
  271. try:
  272. import socksio # noqa
  273. except ImportError: # pragma: no cover
  274. raise ImportError(
  275. "Using SOCKS proxy, but the 'socksio' package is not installed. "
  276. "Make sure to install httpx using `pip install httpx[socks]`."
  277. ) from None
  278. self._pool = httpcore.AsyncSOCKSProxy(
  279. proxy_url=httpcore.URL(
  280. scheme=proxy.url.raw_scheme,
  281. host=proxy.url.raw_host,
  282. port=proxy.url.port,
  283. target=proxy.url.raw_path,
  284. ),
  285. proxy_auth=proxy.raw_auth,
  286. ssl_context=ssl_context,
  287. max_connections=limits.max_connections,
  288. max_keepalive_connections=limits.max_keepalive_connections,
  289. keepalive_expiry=limits.keepalive_expiry,
  290. http1=http1,
  291. http2=http2,
  292. )
  293. else: # pragma: no cover
  294. raise ValueError(
  295. "Proxy protocol must be either 'http', 'https', or 'socks5',"
  296. " but got {proxy.url.scheme!r}."
  297. )
  298. async def __aenter__(self: A) -> A: # Use generics for subclass support.
  299. await self._pool.__aenter__()
  300. return self
  301. async def __aexit__(
  302. self,
  303. exc_type: typing.Optional[typing.Type[BaseException]] = None,
  304. exc_value: typing.Optional[BaseException] = None,
  305. traceback: typing.Optional[TracebackType] = None,
  306. ) -> None:
  307. with map_httpcore_exceptions():
  308. await self._pool.__aexit__(exc_type, exc_value, traceback)
  309. async def handle_async_request(
  310. self,
  311. request: Request,
  312. ) -> Response:
  313. assert isinstance(request.stream, AsyncByteStream)
  314. req = httpcore.Request(
  315. method=request.method,
  316. url=httpcore.URL(
  317. scheme=request.url.raw_scheme,
  318. host=request.url.raw_host,
  319. port=request.url.port,
  320. target=request.url.raw_path,
  321. ),
  322. headers=request.headers.raw,
  323. content=request.stream,
  324. extensions=request.extensions,
  325. )
  326. with map_httpcore_exceptions():
  327. resp = await self._pool.handle_async_request(req)
  328. assert isinstance(resp.stream, typing.AsyncIterable)
  329. return Response(
  330. status_code=resp.status,
  331. headers=resp.headers,
  332. stream=AsyncResponseStream(resp.stream),
  333. extensions=resp.extensions,
  334. )
  335. async def aclose(self) -> None:
  336. await self._pool.aclose()