_models.py 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215
  1. import datetime
  2. import email.message
  3. import json as jsonlib
  4. import typing
  5. import urllib.request
  6. from collections.abc import Mapping
  7. from http.cookiejar import Cookie, CookieJar
  8. from ._content import ByteStream, UnattachedStream, encode_request, encode_response
  9. from ._decoders import (
  10. SUPPORTED_DECODERS,
  11. ByteChunker,
  12. ContentDecoder,
  13. IdentityDecoder,
  14. LineDecoder,
  15. MultiDecoder,
  16. TextChunker,
  17. TextDecoder,
  18. )
  19. from ._exceptions import (
  20. CookieConflict,
  21. HTTPStatusError,
  22. RequestNotRead,
  23. ResponseNotRead,
  24. StreamClosed,
  25. StreamConsumed,
  26. request_context,
  27. )
  28. from ._multipart import get_multipart_boundary_from_content_type
  29. from ._status_codes import codes
  30. from ._types import (
  31. AsyncByteStream,
  32. CookieTypes,
  33. HeaderTypes,
  34. QueryParamTypes,
  35. RequestContent,
  36. RequestData,
  37. RequestExtensions,
  38. RequestFiles,
  39. ResponseContent,
  40. ResponseExtensions,
  41. SyncByteStream,
  42. )
  43. from ._urls import URL
  44. from ._utils import (
  45. is_known_encoding,
  46. normalize_header_key,
  47. normalize_header_value,
  48. obfuscate_sensitive_headers,
  49. parse_content_type_charset,
  50. parse_header_links,
  51. )
  52. class Headers(typing.MutableMapping[str, str]):
  53. """
  54. HTTP headers, as a case-insensitive multi-dict.
  55. """
  56. def __init__(
  57. self,
  58. headers: typing.Optional[HeaderTypes] = None,
  59. encoding: typing.Optional[str] = None,
  60. ) -> None:
  61. if headers is None:
  62. self._list = [] # type: typing.List[typing.Tuple[bytes, bytes, bytes]]
  63. elif isinstance(headers, Headers):
  64. self._list = list(headers._list)
  65. elif isinstance(headers, Mapping):
  66. self._list = [
  67. (
  68. normalize_header_key(k, lower=False, encoding=encoding),
  69. normalize_header_key(k, lower=True, encoding=encoding),
  70. normalize_header_value(v, encoding),
  71. )
  72. for k, v in headers.items()
  73. ]
  74. else:
  75. self._list = [
  76. (
  77. normalize_header_key(k, lower=False, encoding=encoding),
  78. normalize_header_key(k, lower=True, encoding=encoding),
  79. normalize_header_value(v, encoding),
  80. )
  81. for k, v in headers
  82. ]
  83. self._encoding = encoding
  84. @property
  85. def encoding(self) -> str:
  86. """
  87. Header encoding is mandated as ascii, but we allow fallbacks to utf-8
  88. or iso-8859-1.
  89. """
  90. if self._encoding is None:
  91. for encoding in ["ascii", "utf-8"]:
  92. for key, value in self.raw:
  93. try:
  94. key.decode(encoding)
  95. value.decode(encoding)
  96. except UnicodeDecodeError:
  97. break
  98. else:
  99. # The else block runs if 'break' did not occur, meaning
  100. # all values fitted the encoding.
  101. self._encoding = encoding
  102. break
  103. else:
  104. # The ISO-8859-1 encoding covers all 256 code points in a byte,
  105. # so will never raise decode errors.
  106. self._encoding = "iso-8859-1"
  107. return self._encoding
  108. @encoding.setter
  109. def encoding(self, value: str) -> None:
  110. self._encoding = value
  111. @property
  112. def raw(self) -> typing.List[typing.Tuple[bytes, bytes]]:
  113. """
  114. Returns a list of the raw header items, as byte pairs.
  115. """
  116. return [(raw_key, value) for raw_key, _, value in self._list]
  117. def keys(self) -> typing.KeysView[str]:
  118. return {key.decode(self.encoding): None for _, key, value in self._list}.keys()
  119. def values(self) -> typing.ValuesView[str]:
  120. values_dict: typing.Dict[str, str] = {}
  121. for _, key, value in self._list:
  122. str_key = key.decode(self.encoding)
  123. str_value = value.decode(self.encoding)
  124. if str_key in values_dict:
  125. values_dict[str_key] += f", {str_value}"
  126. else:
  127. values_dict[str_key] = str_value
  128. return values_dict.values()
  129. def items(self) -> typing.ItemsView[str, str]:
  130. """
  131. Return `(key, value)` items of headers. Concatenate headers
  132. into a single comma separated value when a key occurs multiple times.
  133. """
  134. values_dict: typing.Dict[str, str] = {}
  135. for _, key, value in self._list:
  136. str_key = key.decode(self.encoding)
  137. str_value = value.decode(self.encoding)
  138. if str_key in values_dict:
  139. values_dict[str_key] += f", {str_value}"
  140. else:
  141. values_dict[str_key] = str_value
  142. return values_dict.items()
  143. def multi_items(self) -> typing.List[typing.Tuple[str, str]]:
  144. """
  145. Return a list of `(key, value)` pairs of headers. Allow multiple
  146. occurrences of the same key without concatenating into a single
  147. comma separated value.
  148. """
  149. return [
  150. (key.decode(self.encoding), value.decode(self.encoding))
  151. for _, key, value in self._list
  152. ]
  153. def get(self, key: str, default: typing.Any = None) -> typing.Any:
  154. """
  155. Return a header value. If multiple occurrences of the header occur
  156. then concatenate them together with commas.
  157. """
  158. try:
  159. return self[key]
  160. except KeyError:
  161. return default
  162. def get_list(self, key: str, split_commas: bool = False) -> typing.List[str]:
  163. """
  164. Return a list of all header values for a given key.
  165. If `split_commas=True` is passed, then any comma separated header
  166. values are split into multiple return strings.
  167. """
  168. get_header_key = key.lower().encode(self.encoding)
  169. values = [
  170. item_value.decode(self.encoding)
  171. for _, item_key, item_value in self._list
  172. if item_key.lower() == get_header_key
  173. ]
  174. if not split_commas:
  175. return values
  176. split_values = []
  177. for value in values:
  178. split_values.extend([item.strip() for item in value.split(",")])
  179. return split_values
  180. def update(self, headers: typing.Optional[HeaderTypes] = None) -> None: # type: ignore
  181. headers = Headers(headers)
  182. for key in headers.keys():
  183. if key in self:
  184. self.pop(key)
  185. self._list.extend(headers._list)
  186. def copy(self) -> "Headers":
  187. return Headers(self, encoding=self.encoding)
  188. def __getitem__(self, key: str) -> str:
  189. """
  190. Return a single header value.
  191. If there are multiple headers with the same key, then we concatenate
  192. them with commas. See: https://tools.ietf.org/html/rfc7230#section-3.2.2
  193. """
  194. normalized_key = key.lower().encode(self.encoding)
  195. items = [
  196. header_value.decode(self.encoding)
  197. for _, header_key, header_value in self._list
  198. if header_key == normalized_key
  199. ]
  200. if items:
  201. return ", ".join(items)
  202. raise KeyError(key)
  203. def __setitem__(self, key: str, value: str) -> None:
  204. """
  205. Set the header `key` to `value`, removing any duplicate entries.
  206. Retains insertion order.
  207. """
  208. set_key = key.encode(self._encoding or "utf-8")
  209. set_value = value.encode(self._encoding or "utf-8")
  210. lookup_key = set_key.lower()
  211. found_indexes = [
  212. idx
  213. for idx, (_, item_key, _) in enumerate(self._list)
  214. if item_key == lookup_key
  215. ]
  216. for idx in reversed(found_indexes[1:]):
  217. del self._list[idx]
  218. if found_indexes:
  219. idx = found_indexes[0]
  220. self._list[idx] = (set_key, lookup_key, set_value)
  221. else:
  222. self._list.append((set_key, lookup_key, set_value))
  223. def __delitem__(self, key: str) -> None:
  224. """
  225. Remove the header `key`.
  226. """
  227. del_key = key.lower().encode(self.encoding)
  228. pop_indexes = [
  229. idx
  230. for idx, (_, item_key, _) in enumerate(self._list)
  231. if item_key.lower() == del_key
  232. ]
  233. if not pop_indexes:
  234. raise KeyError(key)
  235. for idx in reversed(pop_indexes):
  236. del self._list[idx]
  237. def __contains__(self, key: typing.Any) -> bool:
  238. header_key = key.lower().encode(self.encoding)
  239. return header_key in [key for _, key, _ in self._list]
  240. def __iter__(self) -> typing.Iterator[typing.Any]:
  241. return iter(self.keys())
  242. def __len__(self) -> int:
  243. return len(self._list)
  244. def __eq__(self, other: typing.Any) -> bool:
  245. try:
  246. other_headers = Headers(other)
  247. except ValueError:
  248. return False
  249. self_list = [(key, value) for _, key, value in self._list]
  250. other_list = [(key, value) for _, key, value in other_headers._list]
  251. return sorted(self_list) == sorted(other_list)
  252. def __repr__(self) -> str:
  253. class_name = self.__class__.__name__
  254. encoding_str = ""
  255. if self.encoding != "ascii":
  256. encoding_str = f", encoding={self.encoding!r}"
  257. as_list = list(obfuscate_sensitive_headers(self.multi_items()))
  258. as_dict = dict(as_list)
  259. no_duplicate_keys = len(as_dict) == len(as_list)
  260. if no_duplicate_keys:
  261. return f"{class_name}({as_dict!r}{encoding_str})"
  262. return f"{class_name}({as_list!r}{encoding_str})"
  263. class Request:
  264. def __init__(
  265. self,
  266. method: typing.Union[str, bytes],
  267. url: typing.Union["URL", str],
  268. *,
  269. params: typing.Optional[QueryParamTypes] = None,
  270. headers: typing.Optional[HeaderTypes] = None,
  271. cookies: typing.Optional[CookieTypes] = None,
  272. content: typing.Optional[RequestContent] = None,
  273. data: typing.Optional[RequestData] = None,
  274. files: typing.Optional[RequestFiles] = None,
  275. json: typing.Optional[typing.Any] = None,
  276. stream: typing.Union[SyncByteStream, AsyncByteStream, None] = None,
  277. extensions: typing.Optional[RequestExtensions] = None,
  278. ) -> None:
  279. self.method = (
  280. method.decode("ascii").upper()
  281. if isinstance(method, bytes)
  282. else method.upper()
  283. )
  284. self.url = URL(url)
  285. if params is not None:
  286. self.url = self.url.copy_merge_params(params=params)
  287. self.headers = Headers(headers)
  288. self.extensions = {} if extensions is None else extensions
  289. if cookies:
  290. Cookies(cookies).set_cookie_header(self)
  291. if stream is None:
  292. content_type: typing.Optional[str] = self.headers.get("content-type")
  293. headers, stream = encode_request(
  294. content=content,
  295. data=data,
  296. files=files,
  297. json=json,
  298. boundary=get_multipart_boundary_from_content_type(
  299. content_type=content_type.encode(self.headers.encoding)
  300. if content_type
  301. else None
  302. ),
  303. )
  304. self._prepare(headers)
  305. self.stream = stream
  306. # Load the request body, except for streaming content.
  307. if isinstance(stream, ByteStream):
  308. self.read()
  309. else:
  310. # There's an important distinction between `Request(content=...)`,
  311. # and `Request(stream=...)`.
  312. #
  313. # Using `content=...` implies automatically populated `Host` and content
  314. # headers, of either `Content-Length: ...` or `Transfer-Encoding: chunked`.
  315. #
  316. # Using `stream=...` will not automatically include *any*
  317. # auto-populated headers.
  318. #
  319. # As an end-user you don't really need `stream=...`. It's only
  320. # useful when:
  321. #
  322. # * Preserving the request stream when copying requests, eg for redirects.
  323. # * Creating request instances on the *server-side* of the transport API.
  324. self.stream = stream
  325. def _prepare(self, default_headers: typing.Dict[str, str]) -> None:
  326. for key, value in default_headers.items():
  327. # Ignore Transfer-Encoding if the Content-Length has been set explicitly.
  328. if key.lower() == "transfer-encoding" and "Content-Length" in self.headers:
  329. continue
  330. self.headers.setdefault(key, value)
  331. auto_headers: typing.List[typing.Tuple[bytes, bytes]] = []
  332. has_host = "Host" in self.headers
  333. has_content_length = (
  334. "Content-Length" in self.headers or "Transfer-Encoding" in self.headers
  335. )
  336. if not has_host and self.url.host:
  337. auto_headers.append((b"Host", self.url.netloc))
  338. if not has_content_length and self.method in ("POST", "PUT", "PATCH"):
  339. auto_headers.append((b"Content-Length", b"0"))
  340. self.headers = Headers(auto_headers + self.headers.raw)
  341. @property
  342. def content(self) -> bytes:
  343. if not hasattr(self, "_content"):
  344. raise RequestNotRead()
  345. return self._content
  346. def read(self) -> bytes:
  347. """
  348. Read and return the request content.
  349. """
  350. if not hasattr(self, "_content"):
  351. assert isinstance(self.stream, typing.Iterable)
  352. self._content = b"".join(self.stream)
  353. if not isinstance(self.stream, ByteStream):
  354. # If a streaming request has been read entirely into memory, then
  355. # we can replace the stream with a raw bytes implementation,
  356. # to ensure that any non-replayable streams can still be used.
  357. self.stream = ByteStream(self._content)
  358. return self._content
  359. async def aread(self) -> bytes:
  360. """
  361. Read and return the request content.
  362. """
  363. if not hasattr(self, "_content"):
  364. assert isinstance(self.stream, typing.AsyncIterable)
  365. self._content = b"".join([part async for part in self.stream])
  366. if not isinstance(self.stream, ByteStream):
  367. # If a streaming request has been read entirely into memory, then
  368. # we can replace the stream with a raw bytes implementation,
  369. # to ensure that any non-replayable streams can still be used.
  370. self.stream = ByteStream(self._content)
  371. return self._content
  372. def __repr__(self) -> str:
  373. class_name = self.__class__.__name__
  374. url = str(self.url)
  375. return f"<{class_name}({self.method!r}, {url!r})>"
  376. def __getstate__(self) -> typing.Dict[str, typing.Any]:
  377. return {
  378. name: value
  379. for name, value in self.__dict__.items()
  380. if name not in ["extensions", "stream"]
  381. }
  382. def __setstate__(self, state: typing.Dict[str, typing.Any]) -> None:
  383. for name, value in state.items():
  384. setattr(self, name, value)
  385. self.extensions = {}
  386. self.stream = UnattachedStream()
  387. class Response:
  388. def __init__(
  389. self,
  390. status_code: int,
  391. *,
  392. headers: typing.Optional[HeaderTypes] = None,
  393. content: typing.Optional[ResponseContent] = None,
  394. text: typing.Optional[str] = None,
  395. html: typing.Optional[str] = None,
  396. json: typing.Any = None,
  397. stream: typing.Union[SyncByteStream, AsyncByteStream, None] = None,
  398. request: typing.Optional[Request] = None,
  399. extensions: typing.Optional[ResponseExtensions] = None,
  400. history: typing.Optional[typing.List["Response"]] = None,
  401. default_encoding: typing.Union[str, typing.Callable[[bytes], str]] = "utf-8",
  402. ) -> None:
  403. self.status_code = status_code
  404. self.headers = Headers(headers)
  405. self._request: typing.Optional[Request] = request
  406. # When follow_redirects=False and a redirect is received,
  407. # the client will set `response.next_request`.
  408. self.next_request: typing.Optional[Request] = None
  409. self.extensions: ResponseExtensions = {} if extensions is None else extensions
  410. self.history = [] if history is None else list(history)
  411. self.is_closed = False
  412. self.is_stream_consumed = False
  413. self.default_encoding = default_encoding
  414. if stream is None:
  415. headers, stream = encode_response(content, text, html, json)
  416. self._prepare(headers)
  417. self.stream = stream
  418. if isinstance(stream, ByteStream):
  419. # Load the response body, except for streaming content.
  420. self.read()
  421. else:
  422. # There's an important distinction between `Response(content=...)`,
  423. # and `Response(stream=...)`.
  424. #
  425. # Using `content=...` implies automatically populated content headers,
  426. # of either `Content-Length: ...` or `Transfer-Encoding: chunked`.
  427. #
  428. # Using `stream=...` will not automatically include any content headers.
  429. #
  430. # As an end-user you don't really need `stream=...`. It's only
  431. # useful when creating response instances having received a stream
  432. # from the transport API.
  433. self.stream = stream
  434. self._num_bytes_downloaded = 0
  435. def _prepare(self, default_headers: typing.Dict[str, str]) -> None:
  436. for key, value in default_headers.items():
  437. # Ignore Transfer-Encoding if the Content-Length has been set explicitly.
  438. if key.lower() == "transfer-encoding" and "content-length" in self.headers:
  439. continue
  440. self.headers.setdefault(key, value)
  441. @property
  442. def elapsed(self) -> datetime.timedelta:
  443. """
  444. Returns the time taken for the complete request/response
  445. cycle to complete.
  446. """
  447. if not hasattr(self, "_elapsed"):
  448. raise RuntimeError(
  449. "'.elapsed' may only be accessed after the response "
  450. "has been read or closed."
  451. )
  452. return self._elapsed
  453. @elapsed.setter
  454. def elapsed(self, elapsed: datetime.timedelta) -> None:
  455. self._elapsed = elapsed
  456. @property
  457. def request(self) -> Request:
  458. """
  459. Returns the request instance associated to the current response.
  460. """
  461. if self._request is None:
  462. raise RuntimeError(
  463. "The request instance has not been set on this response."
  464. )
  465. return self._request
  466. @request.setter
  467. def request(self, value: Request) -> None:
  468. self._request = value
  469. @property
  470. def http_version(self) -> str:
  471. try:
  472. http_version: bytes = self.extensions["http_version"]
  473. except KeyError:
  474. return "HTTP/1.1"
  475. else:
  476. return http_version.decode("ascii", errors="ignore")
  477. @property
  478. def reason_phrase(self) -> str:
  479. try:
  480. reason_phrase: bytes = self.extensions["reason_phrase"]
  481. except KeyError:
  482. return codes.get_reason_phrase(self.status_code)
  483. else:
  484. return reason_phrase.decode("ascii", errors="ignore")
  485. @property
  486. def url(self) -> URL:
  487. """
  488. Returns the URL for which the request was made.
  489. """
  490. return self.request.url
  491. @property
  492. def content(self) -> bytes:
  493. if not hasattr(self, "_content"):
  494. raise ResponseNotRead()
  495. return self._content
  496. @property
  497. def text(self) -> str:
  498. if not hasattr(self, "_text"):
  499. content = self.content
  500. if not content:
  501. self._text = ""
  502. else:
  503. decoder = TextDecoder(encoding=self.encoding or "utf-8")
  504. self._text = "".join([decoder.decode(self.content), decoder.flush()])
  505. return self._text
  506. @property
  507. def encoding(self) -> typing.Optional[str]:
  508. """
  509. Return an encoding to use for decoding the byte content into text.
  510. The priority for determining this is given by...
  511. * `.encoding = <>` has been set explicitly.
  512. * The encoding as specified by the charset parameter in the Content-Type header.
  513. * The encoding as determined by `default_encoding`, which may either be
  514. a string like "utf-8" indicating the encoding to use, or may be a callable
  515. which enables charset autodetection.
  516. """
  517. if not hasattr(self, "_encoding"):
  518. encoding = self.charset_encoding
  519. if encoding is None or not is_known_encoding(encoding):
  520. if isinstance(self.default_encoding, str):
  521. encoding = self.default_encoding
  522. elif hasattr(self, "_content"):
  523. encoding = self.default_encoding(self._content)
  524. self._encoding = encoding or "utf-8"
  525. return self._encoding
  526. @encoding.setter
  527. def encoding(self, value: str) -> None:
  528. """
  529. Set the encoding to use for decoding the byte content into text.
  530. If the `text` attribute has been accessed, attempting to set the
  531. encoding will throw a ValueError.
  532. """
  533. if hasattr(self, "_text"):
  534. raise ValueError(
  535. "Setting encoding after `text` has been accessed is not allowed."
  536. )
  537. self._encoding = value
  538. @property
  539. def charset_encoding(self) -> typing.Optional[str]:
  540. """
  541. Return the encoding, as specified by the Content-Type header.
  542. """
  543. content_type = self.headers.get("Content-Type")
  544. if content_type is None:
  545. return None
  546. return parse_content_type_charset(content_type)
  547. def _get_content_decoder(self) -> ContentDecoder:
  548. """
  549. Returns a decoder instance which can be used to decode the raw byte
  550. content, depending on the Content-Encoding used in the response.
  551. """
  552. if not hasattr(self, "_decoder"):
  553. decoders: typing.List[ContentDecoder] = []
  554. values = self.headers.get_list("content-encoding", split_commas=True)
  555. for value in values:
  556. value = value.strip().lower()
  557. try:
  558. decoder_cls = SUPPORTED_DECODERS[value]
  559. decoders.append(decoder_cls())
  560. except KeyError:
  561. continue
  562. if len(decoders) == 1:
  563. self._decoder = decoders[0]
  564. elif len(decoders) > 1:
  565. self._decoder = MultiDecoder(children=decoders)
  566. else:
  567. self._decoder = IdentityDecoder()
  568. return self._decoder
  569. @property
  570. def is_informational(self) -> bool:
  571. """
  572. A property which is `True` for 1xx status codes, `False` otherwise.
  573. """
  574. return codes.is_informational(self.status_code)
  575. @property
  576. def is_success(self) -> bool:
  577. """
  578. A property which is `True` for 2xx status codes, `False` otherwise.
  579. """
  580. return codes.is_success(self.status_code)
  581. @property
  582. def is_redirect(self) -> bool:
  583. """
  584. A property which is `True` for 3xx status codes, `False` otherwise.
  585. Note that not all responses with a 3xx status code indicate a URL redirect.
  586. Use `response.has_redirect_location` to determine responses with a properly
  587. formed URL redirection.
  588. """
  589. return codes.is_redirect(self.status_code)
  590. @property
  591. def is_client_error(self) -> bool:
  592. """
  593. A property which is `True` for 4xx status codes, `False` otherwise.
  594. """
  595. return codes.is_client_error(self.status_code)
  596. @property
  597. def is_server_error(self) -> bool:
  598. """
  599. A property which is `True` for 5xx status codes, `False` otherwise.
  600. """
  601. return codes.is_server_error(self.status_code)
  602. @property
  603. def is_error(self) -> bool:
  604. """
  605. A property which is `True` for 4xx and 5xx status codes, `False` otherwise.
  606. """
  607. return codes.is_error(self.status_code)
  608. @property
  609. def has_redirect_location(self) -> bool:
  610. """
  611. Returns True for 3xx responses with a properly formed URL redirection,
  612. `False` otherwise.
  613. """
  614. return (
  615. self.status_code
  616. in (
  617. # 301 (Cacheable redirect. Method may change to GET.)
  618. codes.MOVED_PERMANENTLY,
  619. # 302 (Uncacheable redirect. Method may change to GET.)
  620. codes.FOUND,
  621. # 303 (Client should make a GET or HEAD request.)
  622. codes.SEE_OTHER,
  623. # 307 (Equiv. 302, but retain method)
  624. codes.TEMPORARY_REDIRECT,
  625. # 308 (Equiv. 301, but retain method)
  626. codes.PERMANENT_REDIRECT,
  627. )
  628. and "Location" in self.headers
  629. )
  630. def raise_for_status(self) -> "Response":
  631. """
  632. Raise the `HTTPStatusError` if one occurred.
  633. """
  634. request = self._request
  635. if request is None:
  636. raise RuntimeError(
  637. "Cannot call `raise_for_status` as the request "
  638. "instance has not been set on this response."
  639. )
  640. if self.is_success:
  641. return self
  642. if self.has_redirect_location:
  643. message = (
  644. "{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n"
  645. "Redirect location: '{0.headers[location]}'\n"
  646. "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}"
  647. )
  648. else:
  649. message = (
  650. "{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n"
  651. "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}"
  652. )
  653. status_class = self.status_code // 100
  654. error_types = {
  655. 1: "Informational response",
  656. 3: "Redirect response",
  657. 4: "Client error",
  658. 5: "Server error",
  659. }
  660. error_type = error_types.get(status_class, "Invalid status code")
  661. message = message.format(self, error_type=error_type)
  662. raise HTTPStatusError(message, request=request, response=self)
  663. def json(self, **kwargs: typing.Any) -> typing.Any:
  664. return jsonlib.loads(self.content, **kwargs)
  665. @property
  666. def cookies(self) -> "Cookies":
  667. if not hasattr(self, "_cookies"):
  668. self._cookies = Cookies()
  669. self._cookies.extract_cookies(self)
  670. return self._cookies
  671. @property
  672. def links(self) -> typing.Dict[typing.Optional[str], typing.Dict[str, str]]:
  673. """
  674. Returns the parsed header links of the response, if any
  675. """
  676. header = self.headers.get("link")
  677. ldict = {}
  678. if header:
  679. links = parse_header_links(header)
  680. for link in links:
  681. key = link.get("rel") or link.get("url")
  682. ldict[key] = link
  683. return ldict
  684. @property
  685. def num_bytes_downloaded(self) -> int:
  686. return self._num_bytes_downloaded
  687. def __repr__(self) -> str:
  688. return f"<Response [{self.status_code} {self.reason_phrase}]>"
  689. def __getstate__(self) -> typing.Dict[str, typing.Any]:
  690. return {
  691. name: value
  692. for name, value in self.__dict__.items()
  693. if name not in ["extensions", "stream", "is_closed", "_decoder"]
  694. }
  695. def __setstate__(self, state: typing.Dict[str, typing.Any]) -> None:
  696. for name, value in state.items():
  697. setattr(self, name, value)
  698. self.is_closed = True
  699. self.extensions = {}
  700. self.stream = UnattachedStream()
  701. def read(self) -> bytes:
  702. """
  703. Read and return the response content.
  704. """
  705. if not hasattr(self, "_content"):
  706. self._content = b"".join(self.iter_bytes())
  707. return self._content
  708. def iter_bytes(
  709. self, chunk_size: typing.Optional[int] = None
  710. ) -> typing.Iterator[bytes]:
  711. """
  712. A byte-iterator over the decoded response content.
  713. This allows us to handle gzip, deflate, and brotli encoded responses.
  714. """
  715. if hasattr(self, "_content"):
  716. chunk_size = len(self._content) if chunk_size is None else chunk_size
  717. for i in range(0, len(self._content), max(chunk_size, 1)):
  718. yield self._content[i : i + chunk_size]
  719. else:
  720. decoder = self._get_content_decoder()
  721. chunker = ByteChunker(chunk_size=chunk_size)
  722. with request_context(request=self._request):
  723. for raw_bytes in self.iter_raw():
  724. decoded = decoder.decode(raw_bytes)
  725. for chunk in chunker.decode(decoded):
  726. yield chunk
  727. decoded = decoder.flush()
  728. for chunk in chunker.decode(decoded):
  729. yield chunk # pragma: no cover
  730. for chunk in chunker.flush():
  731. yield chunk
  732. def iter_text(
  733. self, chunk_size: typing.Optional[int] = None
  734. ) -> typing.Iterator[str]:
  735. """
  736. A str-iterator over the decoded response content
  737. that handles both gzip, deflate, etc but also detects the content's
  738. string encoding.
  739. """
  740. decoder = TextDecoder(encoding=self.encoding or "utf-8")
  741. chunker = TextChunker(chunk_size=chunk_size)
  742. with request_context(request=self._request):
  743. for byte_content in self.iter_bytes():
  744. text_content = decoder.decode(byte_content)
  745. for chunk in chunker.decode(text_content):
  746. yield chunk
  747. text_content = decoder.flush()
  748. for chunk in chunker.decode(text_content):
  749. yield chunk # pragma: no cover
  750. for chunk in chunker.flush():
  751. yield chunk
  752. def iter_lines(self) -> typing.Iterator[str]:
  753. decoder = LineDecoder()
  754. with request_context(request=self._request):
  755. for text in self.iter_text():
  756. for line in decoder.decode(text):
  757. yield line
  758. for line in decoder.flush():
  759. yield line
  760. def iter_raw(
  761. self, chunk_size: typing.Optional[int] = None
  762. ) -> typing.Iterator[bytes]:
  763. """
  764. A byte-iterator over the raw response content.
  765. """
  766. if self.is_stream_consumed:
  767. raise StreamConsumed()
  768. if self.is_closed:
  769. raise StreamClosed()
  770. if not isinstance(self.stream, SyncByteStream):
  771. raise RuntimeError("Attempted to call a sync iterator on an async stream.")
  772. self.is_stream_consumed = True
  773. self._num_bytes_downloaded = 0
  774. chunker = ByteChunker(chunk_size=chunk_size)
  775. with request_context(request=self._request):
  776. for raw_stream_bytes in self.stream:
  777. self._num_bytes_downloaded += len(raw_stream_bytes)
  778. for chunk in chunker.decode(raw_stream_bytes):
  779. yield chunk
  780. for chunk in chunker.flush():
  781. yield chunk
  782. self.close()
  783. def close(self) -> None:
  784. """
  785. Close the response and release the connection.
  786. Automatically called if the response body is read to completion.
  787. """
  788. if not isinstance(self.stream, SyncByteStream):
  789. raise RuntimeError("Attempted to call an sync close on an async stream.")
  790. if not self.is_closed:
  791. self.is_closed = True
  792. with request_context(request=self._request):
  793. self.stream.close()
  794. async def aread(self) -> bytes:
  795. """
  796. Read and return the response content.
  797. """
  798. if not hasattr(self, "_content"):
  799. self._content = b"".join([part async for part in self.aiter_bytes()])
  800. return self._content
  801. async def aiter_bytes(
  802. self, chunk_size: typing.Optional[int] = None
  803. ) -> typing.AsyncIterator[bytes]:
  804. """
  805. A byte-iterator over the decoded response content.
  806. This allows us to handle gzip, deflate, and brotli encoded responses.
  807. """
  808. if hasattr(self, "_content"):
  809. chunk_size = len(self._content) if chunk_size is None else chunk_size
  810. for i in range(0, len(self._content), max(chunk_size, 1)):
  811. yield self._content[i : i + chunk_size]
  812. else:
  813. decoder = self._get_content_decoder()
  814. chunker = ByteChunker(chunk_size=chunk_size)
  815. with request_context(request=self._request):
  816. async for raw_bytes in self.aiter_raw():
  817. decoded = decoder.decode(raw_bytes)
  818. for chunk in chunker.decode(decoded):
  819. yield chunk
  820. decoded = decoder.flush()
  821. for chunk in chunker.decode(decoded):
  822. yield chunk # pragma: no cover
  823. for chunk in chunker.flush():
  824. yield chunk
  825. async def aiter_text(
  826. self, chunk_size: typing.Optional[int] = None
  827. ) -> typing.AsyncIterator[str]:
  828. """
  829. A str-iterator over the decoded response content
  830. that handles both gzip, deflate, etc but also detects the content's
  831. string encoding.
  832. """
  833. decoder = TextDecoder(encoding=self.encoding or "utf-8")
  834. chunker = TextChunker(chunk_size=chunk_size)
  835. with request_context(request=self._request):
  836. async for byte_content in self.aiter_bytes():
  837. text_content = decoder.decode(byte_content)
  838. for chunk in chunker.decode(text_content):
  839. yield chunk
  840. text_content = decoder.flush()
  841. for chunk in chunker.decode(text_content):
  842. yield chunk # pragma: no cover
  843. for chunk in chunker.flush():
  844. yield chunk
  845. async def aiter_lines(self) -> typing.AsyncIterator[str]:
  846. decoder = LineDecoder()
  847. with request_context(request=self._request):
  848. async for text in self.aiter_text():
  849. for line in decoder.decode(text):
  850. yield line
  851. for line in decoder.flush():
  852. yield line
  853. async def aiter_raw(
  854. self, chunk_size: typing.Optional[int] = None
  855. ) -> typing.AsyncIterator[bytes]:
  856. """
  857. A byte-iterator over the raw response content.
  858. """
  859. if self.is_stream_consumed:
  860. raise StreamConsumed()
  861. if self.is_closed:
  862. raise StreamClosed()
  863. if not isinstance(self.stream, AsyncByteStream):
  864. raise RuntimeError("Attempted to call an async iterator on an sync stream.")
  865. self.is_stream_consumed = True
  866. self._num_bytes_downloaded = 0
  867. chunker = ByteChunker(chunk_size=chunk_size)
  868. with request_context(request=self._request):
  869. async for raw_stream_bytes in self.stream:
  870. self._num_bytes_downloaded += len(raw_stream_bytes)
  871. for chunk in chunker.decode(raw_stream_bytes):
  872. yield chunk
  873. for chunk in chunker.flush():
  874. yield chunk
  875. await self.aclose()
  876. async def aclose(self) -> None:
  877. """
  878. Close the response and release the connection.
  879. Automatically called if the response body is read to completion.
  880. """
  881. if not isinstance(self.stream, AsyncByteStream):
  882. raise RuntimeError("Attempted to call an async close on an sync stream.")
  883. if not self.is_closed:
  884. self.is_closed = True
  885. with request_context(request=self._request):
  886. await self.stream.aclose()
  887. class Cookies(typing.MutableMapping[str, str]):
  888. """
  889. HTTP Cookies, as a mutable mapping.
  890. """
  891. def __init__(self, cookies: typing.Optional[CookieTypes] = None) -> None:
  892. if cookies is None or isinstance(cookies, dict):
  893. self.jar = CookieJar()
  894. if isinstance(cookies, dict):
  895. for key, value in cookies.items():
  896. self.set(key, value)
  897. elif isinstance(cookies, list):
  898. self.jar = CookieJar()
  899. for key, value in cookies:
  900. self.set(key, value)
  901. elif isinstance(cookies, Cookies):
  902. self.jar = CookieJar()
  903. for cookie in cookies.jar:
  904. self.jar.set_cookie(cookie)
  905. else:
  906. self.jar = cookies
  907. def extract_cookies(self, response: Response) -> None:
  908. """
  909. Loads any cookies based on the response `Set-Cookie` headers.
  910. """
  911. urllib_response = self._CookieCompatResponse(response)
  912. urllib_request = self._CookieCompatRequest(response.request)
  913. self.jar.extract_cookies(urllib_response, urllib_request) # type: ignore
  914. def set_cookie_header(self, request: Request) -> None:
  915. """
  916. Sets an appropriate 'Cookie:' HTTP header on the `Request`.
  917. """
  918. urllib_request = self._CookieCompatRequest(request)
  919. self.jar.add_cookie_header(urllib_request)
  920. def set(self, name: str, value: str, domain: str = "", path: str = "/") -> None:
  921. """
  922. Set a cookie value by name. May optionally include domain and path.
  923. """
  924. kwargs = {
  925. "version": 0,
  926. "name": name,
  927. "value": value,
  928. "port": None,
  929. "port_specified": False,
  930. "domain": domain,
  931. "domain_specified": bool(domain),
  932. "domain_initial_dot": domain.startswith("."),
  933. "path": path,
  934. "path_specified": bool(path),
  935. "secure": False,
  936. "expires": None,
  937. "discard": True,
  938. "comment": None,
  939. "comment_url": None,
  940. "rest": {"HttpOnly": None},
  941. "rfc2109": False,
  942. }
  943. cookie = Cookie(**kwargs) # type: ignore
  944. self.jar.set_cookie(cookie)
  945. def get( # type: ignore
  946. self,
  947. name: str,
  948. default: typing.Optional[str] = None,
  949. domain: typing.Optional[str] = None,
  950. path: typing.Optional[str] = None,
  951. ) -> typing.Optional[str]:
  952. """
  953. Get a cookie by name. May optionally include domain and path
  954. in order to specify exactly which cookie to retrieve.
  955. """
  956. value = None
  957. for cookie in self.jar:
  958. if cookie.name == name:
  959. if domain is None or cookie.domain == domain:
  960. if path is None or cookie.path == path:
  961. if value is not None:
  962. message = f"Multiple cookies exist with name={name}"
  963. raise CookieConflict(message)
  964. value = cookie.value
  965. if value is None:
  966. return default
  967. return value
  968. def delete(
  969. self,
  970. name: str,
  971. domain: typing.Optional[str] = None,
  972. path: typing.Optional[str] = None,
  973. ) -> None:
  974. """
  975. Delete a cookie by name. May optionally include domain and path
  976. in order to specify exactly which cookie to delete.
  977. """
  978. if domain is not None and path is not None:
  979. return self.jar.clear(domain, path, name)
  980. remove = [
  981. cookie
  982. for cookie in self.jar
  983. if cookie.name == name
  984. and (domain is None or cookie.domain == domain)
  985. and (path is None or cookie.path == path)
  986. ]
  987. for cookie in remove:
  988. self.jar.clear(cookie.domain, cookie.path, cookie.name)
  989. def clear(
  990. self, domain: typing.Optional[str] = None, path: typing.Optional[str] = None
  991. ) -> None:
  992. """
  993. Delete all cookies. Optionally include a domain and path in
  994. order to only delete a subset of all the cookies.
  995. """
  996. args = []
  997. if domain is not None:
  998. args.append(domain)
  999. if path is not None:
  1000. assert domain is not None
  1001. args.append(path)
  1002. self.jar.clear(*args)
  1003. def update(self, cookies: typing.Optional[CookieTypes] = None) -> None: # type: ignore
  1004. cookies = Cookies(cookies)
  1005. for cookie in cookies.jar:
  1006. self.jar.set_cookie(cookie)
  1007. def __setitem__(self, name: str, value: str) -> None:
  1008. return self.set(name, value)
  1009. def __getitem__(self, name: str) -> str:
  1010. value = self.get(name)
  1011. if value is None:
  1012. raise KeyError(name)
  1013. return value
  1014. def __delitem__(self, name: str) -> None:
  1015. return self.delete(name)
  1016. def __len__(self) -> int:
  1017. return len(self.jar)
  1018. def __iter__(self) -> typing.Iterator[str]:
  1019. return (cookie.name for cookie in self.jar)
  1020. def __bool__(self) -> bool:
  1021. for _ in self.jar:
  1022. return True
  1023. return False
  1024. def __repr__(self) -> str:
  1025. cookies_repr = ", ".join(
  1026. [
  1027. f"<Cookie {cookie.name}={cookie.value} for {cookie.domain} />"
  1028. for cookie in self.jar
  1029. ]
  1030. )
  1031. return f"<Cookies[{cookies_repr}]>"
  1032. class _CookieCompatRequest(urllib.request.Request):
  1033. """
  1034. Wraps a `Request` instance up in a compatibility interface suitable
  1035. for use with `CookieJar` operations.
  1036. """
  1037. def __init__(self, request: Request) -> None:
  1038. super().__init__(
  1039. url=str(request.url),
  1040. headers=dict(request.headers),
  1041. method=request.method,
  1042. )
  1043. self.request = request
  1044. def add_unredirected_header(self, key: str, value: str) -> None:
  1045. super().add_unredirected_header(key, value)
  1046. self.request.headers[key] = value
  1047. class _CookieCompatResponse:
  1048. """
  1049. Wraps a `Request` instance up in a compatibility interface suitable
  1050. for use with `CookieJar` operations.
  1051. """
  1052. def __init__(self, response: Response) -> None:
  1053. self.response = response
  1054. def info(self) -> email.message.Message:
  1055. info = email.message.Message()
  1056. for key, value in self.response.headers.multi_items():
  1057. # Note that setting `info[key]` here is an "append" operation,
  1058. # not a "replace" operation.
  1059. # https://docs.python.org/3/library/email.compat32-message.html#email.message.Message.__setitem__
  1060. info[key] = value
  1061. return info