_url.py 54 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622
  1. import re
  2. import sys
  3. import warnings
  4. from collections.abc import Mapping, Sequence
  5. from enum import Enum
  6. from functools import _CacheInfo, lru_cache
  7. from ipaddress import ip_address
  8. from typing import (
  9. TYPE_CHECKING,
  10. Any,
  11. NoReturn,
  12. TypedDict,
  13. TypeVar,
  14. Union,
  15. cast,
  16. overload,
  17. )
  18. from urllib.parse import SplitResult, uses_relative
  19. import idna
  20. from multidict import MultiDict, MultiDictProxy, istr
  21. from propcache.api import under_cached_property as cached_property
  22. from ._parse import (
  23. USES_AUTHORITY,
  24. SplitURLType,
  25. make_netloc,
  26. query_to_pairs,
  27. split_netloc,
  28. split_url,
  29. unsplit_result,
  30. )
  31. from ._path import normalize_path, normalize_path_segments
  32. from ._query import (
  33. Query,
  34. QueryVariable,
  35. SimpleQuery,
  36. get_str_query,
  37. get_str_query_from_iterable,
  38. get_str_query_from_sequence_iterable,
  39. )
  40. from ._quoters import (
  41. FRAGMENT_QUOTER,
  42. FRAGMENT_REQUOTER,
  43. PATH_QUOTER,
  44. PATH_REQUOTER,
  45. PATH_SAFE_UNQUOTER,
  46. PATH_UNQUOTER,
  47. QS_UNQUOTER,
  48. QUERY_QUOTER,
  49. QUERY_REQUOTER,
  50. QUOTER,
  51. REQUOTER,
  52. UNQUOTER,
  53. human_quote,
  54. )
  55. DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21}
  56. USES_RELATIVE = frozenset(uses_relative)
  57. # Special schemes https://url.spec.whatwg.org/#special-scheme
  58. # are not allowed to have an empty host https://url.spec.whatwg.org/#url-representation
  59. SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp"))
  60. # reg-name: unreserved / pct-encoded / sub-delims
  61. # this pattern matches anything that is *not* in those classes. and is only used
  62. # on lower-cased ASCII values.
  63. NOT_REG_NAME = re.compile(
  64. r"""
  65. # any character not in the unreserved or sub-delims sets, plus %
  66. # (validated with the additional check for pct-encoded sequences below)
  67. [^a-z0-9\-._~!$&'()*+,;=%]
  68. |
  69. # % only allowed if it is part of a pct-encoded
  70. # sequence of 2 hex digits.
  71. %(?![0-9a-f]{2})
  72. """,
  73. re.VERBOSE,
  74. )
  75. _T = TypeVar("_T")
  76. if sys.version_info >= (3, 11):
  77. from typing import Self
  78. else:
  79. Self = Any
  80. class UndefinedType(Enum):
  81. """Singleton type for use with not set sentinel values."""
  82. _singleton = 0
  83. UNDEFINED = UndefinedType._singleton
  84. class CacheInfo(TypedDict):
  85. """Host encoding cache."""
  86. idna_encode: _CacheInfo
  87. idna_decode: _CacheInfo
  88. ip_address: _CacheInfo
  89. host_validate: _CacheInfo
  90. encode_host: _CacheInfo
  91. class _InternalURLCache(TypedDict, total=False):
  92. _val: SplitURLType
  93. _origin: "URL"
  94. absolute: bool
  95. hash: int
  96. scheme: str
  97. raw_authority: str
  98. authority: str
  99. raw_user: Union[str, None]
  100. user: Union[str, None]
  101. raw_password: Union[str, None]
  102. password: Union[str, None]
  103. raw_host: Union[str, None]
  104. host: Union[str, None]
  105. host_subcomponent: Union[str, None]
  106. host_port_subcomponent: Union[str, None]
  107. port: Union[int, None]
  108. explicit_port: Union[int, None]
  109. raw_path: str
  110. path: str
  111. _parsed_query: list[tuple[str, str]]
  112. query: "MultiDictProxy[str]"
  113. raw_query_string: str
  114. query_string: str
  115. path_qs: str
  116. raw_path_qs: str
  117. raw_fragment: str
  118. fragment: str
  119. raw_parts: tuple[str, ...]
  120. parts: tuple[str, ...]
  121. parent: "URL"
  122. raw_name: str
  123. name: str
  124. raw_suffix: str
  125. suffix: str
  126. raw_suffixes: tuple[str, ...]
  127. suffixes: tuple[str, ...]
  128. def rewrite_module(obj: _T) -> _T:
  129. obj.__module__ = "yarl"
  130. return obj
  131. @lru_cache
  132. def encode_url(url_str: str) -> "URL":
  133. """Parse unencoded URL."""
  134. cache: _InternalURLCache = {}
  135. host: Union[str, None]
  136. scheme, netloc, path, query, fragment = split_url(url_str)
  137. if not netloc: # netloc
  138. host = ""
  139. else:
  140. if ":" in netloc or "@" in netloc or "[" in netloc:
  141. # Complex netloc
  142. username, password, host, port = split_netloc(netloc)
  143. else:
  144. username = password = port = None
  145. host = netloc
  146. if host is None:
  147. if scheme in SCHEME_REQUIRES_HOST:
  148. msg = (
  149. "Invalid URL: host is required for "
  150. f"absolute urls with the {scheme} scheme"
  151. )
  152. raise ValueError(msg)
  153. else:
  154. host = ""
  155. host = _encode_host(host, validate_host=False)
  156. # Remove brackets as host encoder adds back brackets for IPv6 addresses
  157. cache["raw_host"] = host[1:-1] if "[" in host else host
  158. cache["explicit_port"] = port
  159. if password is None and username is None:
  160. # Fast path for URLs without user, password
  161. netloc = host if port is None else f"{host}:{port}"
  162. cache["raw_user"] = None
  163. cache["raw_password"] = None
  164. else:
  165. raw_user = REQUOTER(username) if username else username
  166. raw_password = REQUOTER(password) if password else password
  167. netloc = make_netloc(raw_user, raw_password, host, port)
  168. cache["raw_user"] = raw_user
  169. cache["raw_password"] = raw_password
  170. if path:
  171. path = PATH_REQUOTER(path)
  172. if netloc and "." in path:
  173. path = normalize_path(path)
  174. if query:
  175. query = QUERY_REQUOTER(query)
  176. if fragment:
  177. fragment = FRAGMENT_REQUOTER(fragment)
  178. cache["scheme"] = scheme
  179. cache["raw_path"] = "/" if not path and netloc else path
  180. cache["raw_query_string"] = query
  181. cache["raw_fragment"] = fragment
  182. self = object.__new__(URL)
  183. self._scheme = scheme
  184. self._netloc = netloc
  185. self._path = path
  186. self._query = query
  187. self._fragment = fragment
  188. self._cache = cache
  189. return self
  190. @lru_cache
  191. def pre_encoded_url(url_str: str) -> "URL":
  192. """Parse pre-encoded URL."""
  193. self = object.__new__(URL)
  194. val = split_url(url_str)
  195. self._scheme, self._netloc, self._path, self._query, self._fragment = val
  196. self._cache = {}
  197. return self
  198. @lru_cache
  199. def build_pre_encoded_url(
  200. scheme: str,
  201. authority: str,
  202. user: Union[str, None],
  203. password: Union[str, None],
  204. host: str,
  205. port: Union[int, None],
  206. path: str,
  207. query_string: str,
  208. fragment: str,
  209. ) -> "URL":
  210. """Build a pre-encoded URL from parts."""
  211. self = object.__new__(URL)
  212. self._scheme = scheme
  213. if authority:
  214. self._netloc = authority
  215. elif host:
  216. if port is not None:
  217. port = None if port == DEFAULT_PORTS.get(scheme) else port
  218. if user is None and password is None:
  219. self._netloc = host if port is None else f"{host}:{port}"
  220. else:
  221. self._netloc = make_netloc(user, password, host, port)
  222. else:
  223. self._netloc = ""
  224. self._path = path
  225. self._query = query_string
  226. self._fragment = fragment
  227. self._cache = {}
  228. return self
  229. def from_parts_uncached(
  230. scheme: str, netloc: str, path: str, query: str, fragment: str
  231. ) -> "URL":
  232. """Create a new URL from parts."""
  233. self = object.__new__(URL)
  234. self._scheme = scheme
  235. self._netloc = netloc
  236. self._path = path
  237. self._query = query
  238. self._fragment = fragment
  239. self._cache = {}
  240. return self
  241. from_parts = lru_cache(from_parts_uncached)
  242. @rewrite_module
  243. class URL:
  244. # Don't derive from str
  245. # follow pathlib.Path design
  246. # probably URL will not suffer from pathlib problems:
  247. # it's intended for libraries like aiohttp,
  248. # not to be passed into standard library functions like os.open etc.
  249. # URL grammar (RFC 3986)
  250. # pct-encoded = "%" HEXDIG HEXDIG
  251. # reserved = gen-delims / sub-delims
  252. # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
  253. # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
  254. # / "*" / "+" / "," / ";" / "="
  255. # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
  256. # URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ]
  257. # hier-part = "//" authority path-abempty
  258. # / path-absolute
  259. # / path-rootless
  260. # / path-empty
  261. # scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
  262. # authority = [ userinfo "@" ] host [ ":" port ]
  263. # userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
  264. # host = IP-literal / IPv4address / reg-name
  265. # IP-literal = "[" ( IPv6address / IPvFuture ) "]"
  266. # IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" )
  267. # IPv6address = 6( h16 ":" ) ls32
  268. # / "::" 5( h16 ":" ) ls32
  269. # / [ h16 ] "::" 4( h16 ":" ) ls32
  270. # / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
  271. # / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
  272. # / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
  273. # / [ *4( h16 ":" ) h16 ] "::" ls32
  274. # / [ *5( h16 ":" ) h16 ] "::" h16
  275. # / [ *6( h16 ":" ) h16 ] "::"
  276. # ls32 = ( h16 ":" h16 ) / IPv4address
  277. # ; least-significant 32 bits of address
  278. # h16 = 1*4HEXDIG
  279. # ; 16 bits of address represented in hexadecimal
  280. # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet
  281. # dec-octet = DIGIT ; 0-9
  282. # / %x31-39 DIGIT ; 10-99
  283. # / "1" 2DIGIT ; 100-199
  284. # / "2" %x30-34 DIGIT ; 200-249
  285. # / "25" %x30-35 ; 250-255
  286. # reg-name = *( unreserved / pct-encoded / sub-delims )
  287. # port = *DIGIT
  288. # path = path-abempty ; begins with "/" or is empty
  289. # / path-absolute ; begins with "/" but not "//"
  290. # / path-noscheme ; begins with a non-colon segment
  291. # / path-rootless ; begins with a segment
  292. # / path-empty ; zero characters
  293. # path-abempty = *( "/" segment )
  294. # path-absolute = "/" [ segment-nz *( "/" segment ) ]
  295. # path-noscheme = segment-nz-nc *( "/" segment )
  296. # path-rootless = segment-nz *( "/" segment )
  297. # path-empty = 0<pchar>
  298. # segment = *pchar
  299. # segment-nz = 1*pchar
  300. # segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
  301. # ; non-zero-length segment without any colon ":"
  302. # pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
  303. # query = *( pchar / "/" / "?" )
  304. # fragment = *( pchar / "/" / "?" )
  305. # URI-reference = URI / relative-ref
  306. # relative-ref = relative-part [ "?" query ] [ "#" fragment ]
  307. # relative-part = "//" authority path-abempty
  308. # / path-absolute
  309. # / path-noscheme
  310. # / path-empty
  311. # absolute-URI = scheme ":" hier-part [ "?" query ]
  312. __slots__ = ("_cache", "_scheme", "_netloc", "_path", "_query", "_fragment")
  313. _cache: _InternalURLCache
  314. _scheme: str
  315. _netloc: str
  316. _path: str
  317. _query: str
  318. _fragment: str
  319. def __new__(
  320. cls,
  321. val: Union[str, SplitResult, "URL", UndefinedType] = UNDEFINED,
  322. *,
  323. encoded: bool = False,
  324. strict: Union[bool, None] = None,
  325. ) -> "URL":
  326. if strict is not None: # pragma: no cover
  327. warnings.warn("strict parameter is ignored")
  328. if type(val) is str:
  329. return pre_encoded_url(val) if encoded else encode_url(val)
  330. if type(val) is cls:
  331. return val
  332. if type(val) is SplitResult:
  333. if not encoded:
  334. raise ValueError("Cannot apply decoding to SplitResult")
  335. return from_parts(*val)
  336. if isinstance(val, str):
  337. return pre_encoded_url(str(val)) if encoded else encode_url(str(val))
  338. if val is UNDEFINED:
  339. # Special case for UNDEFINED since it might be unpickling and we do
  340. # not want to cache as the `__set_state__` call would mutate the URL
  341. # object in the `pre_encoded_url` or `encoded_url` caches.
  342. self = object.__new__(URL)
  343. self._scheme = self._netloc = self._path = self._query = self._fragment = ""
  344. self._cache = {}
  345. return self
  346. raise TypeError("Constructor parameter should be str")
  347. @classmethod
  348. def build(
  349. cls,
  350. *,
  351. scheme: str = "",
  352. authority: str = "",
  353. user: Union[str, None] = None,
  354. password: Union[str, None] = None,
  355. host: str = "",
  356. port: Union[int, None] = None,
  357. path: str = "",
  358. query: Union[Query, None] = None,
  359. query_string: str = "",
  360. fragment: str = "",
  361. encoded: bool = False,
  362. ) -> "URL":
  363. """Creates and returns a new URL"""
  364. if authority and (user or password or host or port):
  365. raise ValueError(
  366. 'Can\'t mix "authority" with "user", "password", "host" or "port".'
  367. )
  368. if port is not None and not isinstance(port, int):
  369. raise TypeError(f"The port is required to be int, got {type(port)!r}.")
  370. if port and not host:
  371. raise ValueError('Can\'t build URL with "port" but without "host".')
  372. if query and query_string:
  373. raise ValueError('Only one of "query" or "query_string" should be passed')
  374. if (
  375. scheme is None # type: ignore[redundant-expr]
  376. or authority is None # type: ignore[redundant-expr]
  377. or host is None # type: ignore[redundant-expr]
  378. or path is None # type: ignore[redundant-expr]
  379. or query_string is None # type: ignore[redundant-expr]
  380. or fragment is None
  381. ):
  382. raise TypeError(
  383. 'NoneType is illegal for "scheme", "authority", "host", "path", '
  384. '"query_string", and "fragment" args, use empty string instead.'
  385. )
  386. if query:
  387. query_string = get_str_query(query) or ""
  388. if encoded:
  389. return build_pre_encoded_url(
  390. scheme,
  391. authority,
  392. user,
  393. password,
  394. host,
  395. port,
  396. path,
  397. query_string,
  398. fragment,
  399. )
  400. self = object.__new__(URL)
  401. self._scheme = scheme
  402. _host: Union[str, None] = None
  403. if authority:
  404. user, password, _host, port = split_netloc(authority)
  405. _host = _encode_host(_host, validate_host=False) if _host else ""
  406. elif host:
  407. _host = _encode_host(host, validate_host=True)
  408. else:
  409. self._netloc = ""
  410. if _host is not None:
  411. if port is not None:
  412. port = None if port == DEFAULT_PORTS.get(scheme) else port
  413. if user is None and password is None:
  414. self._netloc = _host if port is None else f"{_host}:{port}"
  415. else:
  416. self._netloc = make_netloc(user, password, _host, port, True)
  417. path = PATH_QUOTER(path) if path else path
  418. if path and self._netloc:
  419. if "." in path:
  420. path = normalize_path(path)
  421. if path[0] != "/":
  422. msg = (
  423. "Path in a URL with authority should "
  424. "start with a slash ('/') if set"
  425. )
  426. raise ValueError(msg)
  427. self._path = path
  428. if not query and query_string:
  429. query_string = QUERY_QUOTER(query_string)
  430. self._query = query_string
  431. self._fragment = FRAGMENT_QUOTER(fragment) if fragment else fragment
  432. self._cache = {}
  433. return self
  434. def __init_subclass__(cls) -> NoReturn:
  435. raise TypeError(f"Inheriting a class {cls!r} from URL is forbidden")
  436. def __str__(self) -> str:
  437. if not self._path and self._netloc and (self._query or self._fragment):
  438. path = "/"
  439. else:
  440. path = self._path
  441. if (port := self.explicit_port) is not None and port == DEFAULT_PORTS.get(
  442. self._scheme
  443. ):
  444. # port normalization - using None for default ports to remove from rendering
  445. # https://datatracker.ietf.org/doc/html/rfc3986.html#section-6.2.3
  446. host = self.host_subcomponent
  447. netloc = make_netloc(self.raw_user, self.raw_password, host, None)
  448. else:
  449. netloc = self._netloc
  450. return unsplit_result(self._scheme, netloc, path, self._query, self._fragment)
  451. def __repr__(self) -> str:
  452. return f"{self.__class__.__name__}('{str(self)}')"
  453. def __bytes__(self) -> bytes:
  454. return str(self).encode("ascii")
  455. def __eq__(self, other: object) -> bool:
  456. if type(other) is not URL:
  457. return NotImplemented
  458. path1 = "/" if not self._path and self._netloc else self._path
  459. path2 = "/" if not other._path and other._netloc else other._path
  460. return (
  461. self._scheme == other._scheme
  462. and self._netloc == other._netloc
  463. and path1 == path2
  464. and self._query == other._query
  465. and self._fragment == other._fragment
  466. )
  467. def __hash__(self) -> int:
  468. if (ret := self._cache.get("hash")) is None:
  469. path = "/" if not self._path and self._netloc else self._path
  470. ret = self._cache["hash"] = hash(
  471. (self._scheme, self._netloc, path, self._query, self._fragment)
  472. )
  473. return ret
  474. def __le__(self, other: object) -> bool:
  475. if type(other) is not URL:
  476. return NotImplemented
  477. return self._val <= other._val
  478. def __lt__(self, other: object) -> bool:
  479. if type(other) is not URL:
  480. return NotImplemented
  481. return self._val < other._val
  482. def __ge__(self, other: object) -> bool:
  483. if type(other) is not URL:
  484. return NotImplemented
  485. return self._val >= other._val
  486. def __gt__(self, other: object) -> bool:
  487. if type(other) is not URL:
  488. return NotImplemented
  489. return self._val > other._val
  490. def __truediv__(self, name: str) -> "URL":
  491. if not isinstance(name, str):
  492. return NotImplemented # type: ignore[unreachable]
  493. return self._make_child((str(name),))
  494. def __mod__(self, query: Query) -> "URL":
  495. return self.update_query(query)
  496. def __bool__(self) -> bool:
  497. return bool(self._netloc or self._path or self._query or self._fragment)
  498. def __getstate__(self) -> tuple[SplitResult]:
  499. return (tuple.__new__(SplitResult, self._val),)
  500. def __setstate__(
  501. self, state: Union[tuple[SplitURLType], tuple[None, _InternalURLCache]]
  502. ) -> None:
  503. if state[0] is None and isinstance(state[1], dict):
  504. # default style pickle
  505. val = state[1]["_val"]
  506. else:
  507. unused: list[object]
  508. val, *unused = state
  509. self._scheme, self._netloc, self._path, self._query, self._fragment = val
  510. self._cache = {}
  511. def _cache_netloc(self) -> None:
  512. """Cache the netloc parts of the URL."""
  513. c = self._cache
  514. split_loc = split_netloc(self._netloc)
  515. c["raw_user"], c["raw_password"], c["raw_host"], c["explicit_port"] = split_loc
  516. def is_absolute(self) -> bool:
  517. """A check for absolute URLs.
  518. Return True for absolute ones (having scheme or starting
  519. with //), False otherwise.
  520. Is is preferred to call the .absolute property instead
  521. as it is cached.
  522. """
  523. return self.absolute
  524. def is_default_port(self) -> bool:
  525. """A check for default port.
  526. Return True if port is default for specified scheme,
  527. e.g. 'http://python.org' or 'http://python.org:80', False
  528. otherwise.
  529. Return False for relative URLs.
  530. """
  531. if (explicit := self.explicit_port) is None:
  532. # If the explicit port is None, then the URL must be
  533. # using the default port unless its a relative URL
  534. # which does not have an implicit port / default port
  535. return self._netloc != ""
  536. return explicit == DEFAULT_PORTS.get(self._scheme)
  537. def origin(self) -> "URL":
  538. """Return an URL with scheme, host and port parts only.
  539. user, password, path, query and fragment are removed.
  540. """
  541. # TODO: add a keyword-only option for keeping user/pass maybe?
  542. return self._origin
  543. @cached_property
  544. def _val(self) -> SplitURLType:
  545. return (self._scheme, self._netloc, self._path, self._query, self._fragment)
  546. @cached_property
  547. def _origin(self) -> "URL":
  548. """Return an URL with scheme, host and port parts only.
  549. user, password, path, query and fragment are removed.
  550. """
  551. if not (netloc := self._netloc):
  552. raise ValueError("URL should be absolute")
  553. if not (scheme := self._scheme):
  554. raise ValueError("URL should have scheme")
  555. if "@" in netloc:
  556. encoded_host = self.host_subcomponent
  557. netloc = make_netloc(None, None, encoded_host, self.explicit_port)
  558. elif not self._path and not self._query and not self._fragment:
  559. return self
  560. return from_parts(scheme, netloc, "", "", "")
  561. def relative(self) -> "URL":
  562. """Return a relative part of the URL.
  563. scheme, user, password, host and port are removed.
  564. """
  565. if not self._netloc:
  566. raise ValueError("URL should be absolute")
  567. return from_parts("", "", self._path, self._query, self._fragment)
  568. @cached_property
  569. def absolute(self) -> bool:
  570. """A check for absolute URLs.
  571. Return True for absolute ones (having scheme or starting
  572. with //), False otherwise.
  573. """
  574. # `netloc`` is an empty string for relative URLs
  575. # Checking `netloc` is faster than checking `hostname`
  576. # because `hostname` is a property that does some extra work
  577. # to parse the host from the `netloc`
  578. return self._netloc != ""
  579. @cached_property
  580. def scheme(self) -> str:
  581. """Scheme for absolute URLs.
  582. Empty string for relative URLs or URLs starting with //
  583. """
  584. return self._scheme
  585. @cached_property
  586. def raw_authority(self) -> str:
  587. """Encoded authority part of URL.
  588. Empty string for relative URLs.
  589. """
  590. return self._netloc
  591. @cached_property
  592. def authority(self) -> str:
  593. """Decoded authority part of URL.
  594. Empty string for relative URLs.
  595. """
  596. return make_netloc(self.user, self.password, self.host, self.port)
  597. @cached_property
  598. def raw_user(self) -> Union[str, None]:
  599. """Encoded user part of URL.
  600. None if user is missing.
  601. """
  602. # not .username
  603. self._cache_netloc()
  604. return self._cache["raw_user"]
  605. @cached_property
  606. def user(self) -> Union[str, None]:
  607. """Decoded user part of URL.
  608. None if user is missing.
  609. """
  610. if (raw_user := self.raw_user) is None:
  611. return None
  612. return UNQUOTER(raw_user)
  613. @cached_property
  614. def raw_password(self) -> Union[str, None]:
  615. """Encoded password part of URL.
  616. None if password is missing.
  617. """
  618. self._cache_netloc()
  619. return self._cache["raw_password"]
  620. @cached_property
  621. def password(self) -> Union[str, None]:
  622. """Decoded password part of URL.
  623. None if password is missing.
  624. """
  625. if (raw_password := self.raw_password) is None:
  626. return None
  627. return UNQUOTER(raw_password)
  628. @cached_property
  629. def raw_host(self) -> Union[str, None]:
  630. """Encoded host part of URL.
  631. None for relative URLs.
  632. When working with IPv6 addresses, use the `host_subcomponent` property instead
  633. as it will return the host subcomponent with brackets.
  634. """
  635. # Use host instead of hostname for sake of shortness
  636. # May add .hostname prop later
  637. self._cache_netloc()
  638. return self._cache["raw_host"]
  639. @cached_property
  640. def host(self) -> Union[str, None]:
  641. """Decoded host part of URL.
  642. None for relative URLs.
  643. """
  644. if (raw := self.raw_host) is None:
  645. return None
  646. if raw and raw[-1].isdigit() or ":" in raw:
  647. # IP addresses are never IDNA encoded
  648. return raw
  649. return _idna_decode(raw)
  650. @cached_property
  651. def host_subcomponent(self) -> Union[str, None]:
  652. """Return the host subcomponent part of URL.
  653. None for relative URLs.
  654. https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
  655. `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`
  656. Examples:
  657. - `http://example.com:8080` -> `example.com`
  658. - `http://example.com:80` -> `example.com`
  659. - `https://127.0.0.1:8443` -> `127.0.0.1`
  660. - `https://[::1]:8443` -> `[::1]`
  661. - `http://[::1]` -> `[::1]`
  662. """
  663. if (raw := self.raw_host) is None:
  664. return None
  665. return f"[{raw}]" if ":" in raw else raw
  666. @cached_property
  667. def host_port_subcomponent(self) -> Union[str, None]:
  668. """Return the host and port subcomponent part of URL.
  669. Trailing dots are removed from the host part.
  670. This value is suitable for use in the Host header of an HTTP request.
  671. None for relative URLs.
  672. https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
  673. `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`
  674. https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.3
  675. port = *DIGIT
  676. Examples:
  677. - `http://example.com:8080` -> `example.com:8080`
  678. - `http://example.com:80` -> `example.com`
  679. - `http://example.com.:80` -> `example.com`
  680. - `https://127.0.0.1:8443` -> `127.0.0.1:8443`
  681. - `https://[::1]:8443` -> `[::1]:8443`
  682. - `http://[::1]` -> `[::1]`
  683. """
  684. if (raw := self.raw_host) is None:
  685. return None
  686. if raw[-1] == ".":
  687. # Remove all trailing dots from the netloc as while
  688. # they are valid FQDNs in DNS, TLS validation fails.
  689. # See https://github.com/aio-libs/aiohttp/issues/3636.
  690. # To avoid string manipulation we only call rstrip if
  691. # the last character is a dot.
  692. raw = raw.rstrip(".")
  693. port = self.explicit_port
  694. if port is None or port == DEFAULT_PORTS.get(self._scheme):
  695. return f"[{raw}]" if ":" in raw else raw
  696. return f"[{raw}]:{port}" if ":" in raw else f"{raw}:{port}"
  697. @cached_property
  698. def port(self) -> Union[int, None]:
  699. """Port part of URL, with scheme-based fallback.
  700. None for relative URLs or URLs without explicit port and
  701. scheme without default port substitution.
  702. """
  703. if (explicit_port := self.explicit_port) is not None:
  704. return explicit_port
  705. return DEFAULT_PORTS.get(self._scheme)
  706. @cached_property
  707. def explicit_port(self) -> Union[int, None]:
  708. """Port part of URL, without scheme-based fallback.
  709. None for relative URLs or URLs without explicit port.
  710. """
  711. self._cache_netloc()
  712. return self._cache["explicit_port"]
  713. @cached_property
  714. def raw_path(self) -> str:
  715. """Encoded path of URL.
  716. / for absolute URLs without path part.
  717. """
  718. return self._path if self._path or not self._netloc else "/"
  719. @cached_property
  720. def path(self) -> str:
  721. """Decoded path of URL.
  722. / for absolute URLs without path part.
  723. """
  724. return PATH_UNQUOTER(self._path) if self._path else "/" if self._netloc else ""
  725. @cached_property
  726. def path_safe(self) -> str:
  727. """Decoded path of URL.
  728. / for absolute URLs without path part.
  729. / (%2F) and % (%25) are not decoded
  730. """
  731. if self._path:
  732. return PATH_SAFE_UNQUOTER(self._path)
  733. return "/" if self._netloc else ""
  734. @cached_property
  735. def _parsed_query(self) -> list[tuple[str, str]]:
  736. """Parse query part of URL."""
  737. return query_to_pairs(self._query)
  738. @cached_property
  739. def query(self) -> "MultiDictProxy[str]":
  740. """A MultiDictProxy representing parsed query parameters in decoded
  741. representation.
  742. Empty value if URL has no query part.
  743. """
  744. return MultiDictProxy(MultiDict(self._parsed_query))
  745. @cached_property
  746. def raw_query_string(self) -> str:
  747. """Encoded query part of URL.
  748. Empty string if query is missing.
  749. """
  750. return self._query
  751. @cached_property
  752. def query_string(self) -> str:
  753. """Decoded query part of URL.
  754. Empty string if query is missing.
  755. """
  756. return QS_UNQUOTER(self._query) if self._query else ""
  757. @cached_property
  758. def path_qs(self) -> str:
  759. """Decoded path of URL with query."""
  760. return self.path if not (q := self.query_string) else f"{self.path}?{q}"
  761. @cached_property
  762. def raw_path_qs(self) -> str:
  763. """Encoded path of URL with query."""
  764. if q := self._query:
  765. return f"{self._path}?{q}" if self._path or not self._netloc else f"/?{q}"
  766. return self._path if self._path or not self._netloc else "/"
  767. @cached_property
  768. def raw_fragment(self) -> str:
  769. """Encoded fragment part of URL.
  770. Empty string if fragment is missing.
  771. """
  772. return self._fragment
  773. @cached_property
  774. def fragment(self) -> str:
  775. """Decoded fragment part of URL.
  776. Empty string if fragment is missing.
  777. """
  778. return UNQUOTER(self._fragment) if self._fragment else ""
  779. @cached_property
  780. def raw_parts(self) -> tuple[str, ...]:
  781. """A tuple containing encoded *path* parts.
  782. ('/',) for absolute URLs if *path* is missing.
  783. """
  784. path = self._path
  785. if self._netloc:
  786. return ("/", *path[1:].split("/")) if path else ("/",)
  787. if path and path[0] == "/":
  788. return ("/", *path[1:].split("/"))
  789. return tuple(path.split("/"))
  790. @cached_property
  791. def parts(self) -> tuple[str, ...]:
  792. """A tuple containing decoded *path* parts.
  793. ('/',) for absolute URLs if *path* is missing.
  794. """
  795. return tuple(UNQUOTER(part) for part in self.raw_parts)
  796. @cached_property
  797. def parent(self) -> "URL":
  798. """A new URL with last part of path removed and cleaned up query and
  799. fragment.
  800. """
  801. path = self._path
  802. if not path or path == "/":
  803. if self._fragment or self._query:
  804. return from_parts(self._scheme, self._netloc, path, "", "")
  805. return self
  806. parts = path.split("/")
  807. return from_parts(self._scheme, self._netloc, "/".join(parts[:-1]), "", "")
  808. @cached_property
  809. def raw_name(self) -> str:
  810. """The last part of raw_parts."""
  811. parts = self.raw_parts
  812. if not self._netloc:
  813. return parts[-1]
  814. parts = parts[1:]
  815. return parts[-1] if parts else ""
  816. @cached_property
  817. def name(self) -> str:
  818. """The last part of parts."""
  819. return UNQUOTER(self.raw_name)
  820. @cached_property
  821. def raw_suffix(self) -> str:
  822. name = self.raw_name
  823. i = name.rfind(".")
  824. return name[i:] if 0 < i < len(name) - 1 else ""
  825. @cached_property
  826. def suffix(self) -> str:
  827. return UNQUOTER(self.raw_suffix)
  828. @cached_property
  829. def raw_suffixes(self) -> tuple[str, ...]:
  830. name = self.raw_name
  831. if name.endswith("."):
  832. return ()
  833. name = name.lstrip(".")
  834. return tuple("." + suffix for suffix in name.split(".")[1:])
  835. @cached_property
  836. def suffixes(self) -> tuple[str, ...]:
  837. return tuple(UNQUOTER(suffix) for suffix in self.raw_suffixes)
  838. def _make_child(self, paths: "Sequence[str]", encoded: bool = False) -> "URL":
  839. """
  840. add paths to self._path, accounting for absolute vs relative paths,
  841. keep existing, but do not create new, empty segments
  842. """
  843. parsed: list[str] = []
  844. needs_normalize: bool = False
  845. for idx, path in enumerate(reversed(paths)):
  846. # empty segment of last is not removed
  847. last = idx == 0
  848. if path and path[0] == "/":
  849. raise ValueError(
  850. f"Appending path {path!r} starting from slash is forbidden"
  851. )
  852. # We need to quote the path if it is not already encoded
  853. # This cannot be done at the end because the existing
  854. # path is already quoted and we do not want to double quote
  855. # the existing path.
  856. path = path if encoded else PATH_QUOTER(path)
  857. needs_normalize |= "." in path
  858. segments = path.split("/")
  859. segments.reverse()
  860. # remove trailing empty segment for all but the last path
  861. parsed += segments[1:] if not last and segments[0] == "" else segments
  862. if (path := self._path) and (old_segments := path.split("/")):
  863. # If the old path ends with a slash, the last segment is an empty string
  864. # and should be removed before adding the new path segments.
  865. old = old_segments[:-1] if old_segments[-1] == "" else old_segments
  866. old.reverse()
  867. parsed += old
  868. # If the netloc is present, inject a leading slash when adding a
  869. # path to an absolute URL where there was none before.
  870. if (netloc := self._netloc) and parsed and parsed[-1] != "":
  871. parsed.append("")
  872. parsed.reverse()
  873. if not netloc or not needs_normalize:
  874. return from_parts(self._scheme, netloc, "/".join(parsed), "", "")
  875. path = "/".join(normalize_path_segments(parsed))
  876. # If normalizing the path segments removed the leading slash, add it back.
  877. if path and path[0] != "/":
  878. path = f"/{path}"
  879. return from_parts(self._scheme, netloc, path, "", "")
  880. def with_scheme(self, scheme: str) -> "URL":
  881. """Return a new URL with scheme replaced."""
  882. # N.B. doesn't cleanup query/fragment
  883. if not isinstance(scheme, str):
  884. raise TypeError("Invalid scheme type")
  885. lower_scheme = scheme.lower()
  886. netloc = self._netloc
  887. if not netloc and lower_scheme in SCHEME_REQUIRES_HOST:
  888. msg = (
  889. "scheme replacement is not allowed for "
  890. f"relative URLs for the {lower_scheme} scheme"
  891. )
  892. raise ValueError(msg)
  893. return from_parts(lower_scheme, netloc, self._path, self._query, self._fragment)
  894. def with_user(self, user: Union[str, None]) -> "URL":
  895. """Return a new URL with user replaced.
  896. Autoencode user if needed.
  897. Clear user/password if user is None.
  898. """
  899. # N.B. doesn't cleanup query/fragment
  900. if user is None:
  901. password = None
  902. elif isinstance(user, str):
  903. user = QUOTER(user)
  904. password = self.raw_password
  905. else:
  906. raise TypeError("Invalid user type")
  907. if not (netloc := self._netloc):
  908. raise ValueError("user replacement is not allowed for relative URLs")
  909. encoded_host = self.host_subcomponent or ""
  910. netloc = make_netloc(user, password, encoded_host, self.explicit_port)
  911. return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
  912. def with_password(self, password: Union[str, None]) -> "URL":
  913. """Return a new URL with password replaced.
  914. Autoencode password if needed.
  915. Clear password if argument is None.
  916. """
  917. # N.B. doesn't cleanup query/fragment
  918. if password is None:
  919. pass
  920. elif isinstance(password, str):
  921. password = QUOTER(password)
  922. else:
  923. raise TypeError("Invalid password type")
  924. if not (netloc := self._netloc):
  925. raise ValueError("password replacement is not allowed for relative URLs")
  926. encoded_host = self.host_subcomponent or ""
  927. port = self.explicit_port
  928. netloc = make_netloc(self.raw_user, password, encoded_host, port)
  929. return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
  930. def with_host(self, host: str) -> "URL":
  931. """Return a new URL with host replaced.
  932. Autoencode host if needed.
  933. Changing host for relative URLs is not allowed, use .join()
  934. instead.
  935. """
  936. # N.B. doesn't cleanup query/fragment
  937. if not isinstance(host, str):
  938. raise TypeError("Invalid host type")
  939. if not (netloc := self._netloc):
  940. raise ValueError("host replacement is not allowed for relative URLs")
  941. if not host:
  942. raise ValueError("host removing is not allowed")
  943. encoded_host = _encode_host(host, validate_host=True) if host else ""
  944. port = self.explicit_port
  945. netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)
  946. return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
  947. def with_port(self, port: Union[int, None]) -> "URL":
  948. """Return a new URL with port replaced.
  949. Clear port to default if None is passed.
  950. """
  951. # N.B. doesn't cleanup query/fragment
  952. if port is not None:
  953. if isinstance(port, bool) or not isinstance(port, int):
  954. raise TypeError(f"port should be int or None, got {type(port)}")
  955. if not (0 <= port <= 65535):
  956. raise ValueError(f"port must be between 0 and 65535, got {port}")
  957. if not (netloc := self._netloc):
  958. raise ValueError("port replacement is not allowed for relative URLs")
  959. encoded_host = self.host_subcomponent or ""
  960. netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)
  961. return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
  962. def with_path(
  963. self,
  964. path: str,
  965. *,
  966. encoded: bool = False,
  967. keep_query: bool = False,
  968. keep_fragment: bool = False,
  969. ) -> "URL":
  970. """Return a new URL with path replaced."""
  971. netloc = self._netloc
  972. if not encoded:
  973. path = PATH_QUOTER(path)
  974. if netloc:
  975. path = normalize_path(path) if "." in path else path
  976. if path and path[0] != "/":
  977. path = f"/{path}"
  978. query = self._query if keep_query else ""
  979. fragment = self._fragment if keep_fragment else ""
  980. return from_parts(self._scheme, netloc, path, query, fragment)
  981. @overload
  982. def with_query(self, query: Query) -> "URL": ...
  983. @overload
  984. def with_query(self, **kwargs: QueryVariable) -> "URL": ...
  985. def with_query(self, *args: Any, **kwargs: Any) -> "URL":
  986. """Return a new URL with query part replaced.
  987. Accepts any Mapping (e.g. dict, multidict.MultiDict instances)
  988. or str, autoencode the argument if needed.
  989. A sequence of (key, value) pairs is supported as well.
  990. It also can take an arbitrary number of keyword arguments.
  991. Clear query if None is passed.
  992. """
  993. # N.B. doesn't cleanup query/fragment
  994. query = get_str_query(*args, **kwargs) or ""
  995. return from_parts_uncached(
  996. self._scheme, self._netloc, self._path, query, self._fragment
  997. )
  998. @overload
  999. def extend_query(self, query: Query) -> "URL": ...
  1000. @overload
  1001. def extend_query(self, **kwargs: QueryVariable) -> "URL": ...
  1002. def extend_query(self, *args: Any, **kwargs: Any) -> "URL":
  1003. """Return a new URL with query part combined with the existing.
  1004. This method will not remove existing query parameters.
  1005. Example:
  1006. >>> url = URL('http://example.com/?a=1&b=2')
  1007. >>> url.extend_query(a=3, c=4)
  1008. URL('http://example.com/?a=1&b=2&a=3&c=4')
  1009. """
  1010. if not (new_query := get_str_query(*args, **kwargs)):
  1011. return self
  1012. if query := self._query:
  1013. # both strings are already encoded so we can use a simple
  1014. # string join
  1015. query += new_query if query[-1] == "&" else f"&{new_query}"
  1016. else:
  1017. query = new_query
  1018. return from_parts_uncached(
  1019. self._scheme, self._netloc, self._path, query, self._fragment
  1020. )
  1021. @overload
  1022. def update_query(self, query: Query) -> "URL": ...
  1023. @overload
  1024. def update_query(self, **kwargs: QueryVariable) -> "URL": ...
  1025. def update_query(self, *args: Any, **kwargs: Any) -> "URL":
  1026. """Return a new URL with query part updated.
  1027. This method will overwrite existing query parameters.
  1028. Example:
  1029. >>> url = URL('http://example.com/?a=1&b=2')
  1030. >>> url.update_query(a=3, c=4)
  1031. URL('http://example.com/?a=3&b=2&c=4')
  1032. """
  1033. in_query: Union[
  1034. str,
  1035. Mapping[str, QueryVariable],
  1036. Sequence[tuple[Union[str, istr], SimpleQuery]],
  1037. None,
  1038. ]
  1039. if kwargs:
  1040. if args:
  1041. msg = "Either kwargs or single query parameter must be present"
  1042. raise ValueError(msg)
  1043. in_query = kwargs
  1044. elif len(args) == 1:
  1045. in_query = args[0]
  1046. else:
  1047. raise ValueError("Either kwargs or single query parameter must be present")
  1048. if in_query is None:
  1049. query = ""
  1050. elif not in_query:
  1051. query = self._query
  1052. elif isinstance(in_query, Mapping):
  1053. qm: MultiDict[QueryVariable] = MultiDict(self._parsed_query)
  1054. qm.update(in_query)
  1055. query = get_str_query_from_sequence_iterable(qm.items())
  1056. elif isinstance(in_query, str):
  1057. qstr: MultiDict[str] = MultiDict(self._parsed_query)
  1058. qstr.update(query_to_pairs(in_query))
  1059. query = get_str_query_from_iterable(qstr.items())
  1060. elif isinstance(in_query, (bytes, bytearray, memoryview)):
  1061. msg = "Invalid query type: bytes, bytearray and memoryview are forbidden"
  1062. raise TypeError(msg)
  1063. elif isinstance(in_query, Sequence):
  1064. # We don't expect sequence values if we're given a list of pairs
  1065. # already; only mappings like builtin `dict` which can't have the
  1066. # same key pointing to multiple values are allowed to use
  1067. # `_query_seq_pairs`.
  1068. if TYPE_CHECKING:
  1069. in_query = cast(
  1070. Sequence[tuple[Union[str, istr], SimpleQuery]], in_query
  1071. )
  1072. qs: MultiDict[SimpleQuery] = MultiDict(self._parsed_query)
  1073. qs.update(in_query)
  1074. query = get_str_query_from_iterable(qs.items())
  1075. else:
  1076. raise TypeError(
  1077. "Invalid query type: only str, mapping or "
  1078. "sequence of (key, value) pairs is allowed"
  1079. )
  1080. return from_parts_uncached(
  1081. self._scheme, self._netloc, self._path, query, self._fragment
  1082. )
  1083. def without_query_params(self, *query_params: str) -> "URL":
  1084. """Remove some keys from query part and return new URL."""
  1085. params_to_remove = set(query_params) & self.query.keys()
  1086. if not params_to_remove:
  1087. return self
  1088. return self.with_query(
  1089. tuple(
  1090. (name, value)
  1091. for name, value in self.query.items()
  1092. if name not in params_to_remove
  1093. )
  1094. )
  1095. def with_fragment(self, fragment: Union[str, None]) -> "URL":
  1096. """Return a new URL with fragment replaced.
  1097. Autoencode fragment if needed.
  1098. Clear fragment to default if None is passed.
  1099. """
  1100. # N.B. doesn't cleanup query/fragment
  1101. if fragment is None:
  1102. raw_fragment = ""
  1103. elif not isinstance(fragment, str):
  1104. raise TypeError("Invalid fragment type")
  1105. else:
  1106. raw_fragment = FRAGMENT_QUOTER(fragment)
  1107. if self._fragment == raw_fragment:
  1108. return self
  1109. return from_parts(
  1110. self._scheme, self._netloc, self._path, self._query, raw_fragment
  1111. )
  1112. def with_name(
  1113. self,
  1114. name: str,
  1115. *,
  1116. keep_query: bool = False,
  1117. keep_fragment: bool = False,
  1118. ) -> "URL":
  1119. """Return a new URL with name (last part of path) replaced.
  1120. Query and fragment parts are cleaned up.
  1121. Name is encoded if needed.
  1122. """
  1123. # N.B. DOES cleanup query/fragment
  1124. if not isinstance(name, str):
  1125. raise TypeError("Invalid name type")
  1126. if "/" in name:
  1127. raise ValueError("Slash in name is not allowed")
  1128. name = PATH_QUOTER(name)
  1129. if name in (".", ".."):
  1130. raise ValueError(". and .. values are forbidden")
  1131. parts = list(self.raw_parts)
  1132. if netloc := self._netloc:
  1133. if len(parts) == 1:
  1134. parts.append(name)
  1135. else:
  1136. parts[-1] = name
  1137. parts[0] = "" # replace leading '/'
  1138. else:
  1139. parts[-1] = name
  1140. if parts[0] == "/":
  1141. parts[0] = "" # replace leading '/'
  1142. query = self._query if keep_query else ""
  1143. fragment = self._fragment if keep_fragment else ""
  1144. return from_parts(self._scheme, netloc, "/".join(parts), query, fragment)
  1145. def with_suffix(
  1146. self,
  1147. suffix: str,
  1148. *,
  1149. keep_query: bool = False,
  1150. keep_fragment: bool = False,
  1151. ) -> "URL":
  1152. """Return a new URL with suffix (file extension of name) replaced.
  1153. Query and fragment parts are cleaned up.
  1154. suffix is encoded if needed.
  1155. """
  1156. if not isinstance(suffix, str):
  1157. raise TypeError("Invalid suffix type")
  1158. if suffix and not suffix[0] == "." or suffix == "." or "/" in suffix:
  1159. raise ValueError(f"Invalid suffix {suffix!r}")
  1160. name = self.raw_name
  1161. if not name:
  1162. raise ValueError(f"{self!r} has an empty name")
  1163. old_suffix = self.raw_suffix
  1164. suffix = PATH_QUOTER(suffix)
  1165. name = name + suffix if not old_suffix else name[: -len(old_suffix)] + suffix
  1166. if name in (".", ".."):
  1167. raise ValueError(". and .. values are forbidden")
  1168. parts = list(self.raw_parts)
  1169. if netloc := self._netloc:
  1170. if len(parts) == 1:
  1171. parts.append(name)
  1172. else:
  1173. parts[-1] = name
  1174. parts[0] = "" # replace leading '/'
  1175. else:
  1176. parts[-1] = name
  1177. if parts[0] == "/":
  1178. parts[0] = "" # replace leading '/'
  1179. query = self._query if keep_query else ""
  1180. fragment = self._fragment if keep_fragment else ""
  1181. return from_parts(self._scheme, netloc, "/".join(parts), query, fragment)
  1182. def join(self, url: "URL") -> "URL":
  1183. """Join URLs
  1184. Construct a full (“absolute”) URL by combining a “base URL”
  1185. (self) with another URL (url).
  1186. Informally, this uses components of the base URL, in
  1187. particular the addressing scheme, the network location and
  1188. (part of) the path, to provide missing components in the
  1189. relative URL.
  1190. """
  1191. if type(url) is not URL:
  1192. raise TypeError("url should be URL")
  1193. scheme = url._scheme or self._scheme
  1194. if scheme != self._scheme or scheme not in USES_RELATIVE:
  1195. return url
  1196. # scheme is in uses_authority as uses_authority is a superset of uses_relative
  1197. if (join_netloc := url._netloc) and scheme in USES_AUTHORITY:
  1198. return from_parts(scheme, join_netloc, url._path, url._query, url._fragment)
  1199. orig_path = self._path
  1200. if join_path := url._path:
  1201. if join_path[0] == "/":
  1202. path = join_path
  1203. elif not orig_path:
  1204. path = f"/{join_path}"
  1205. elif orig_path[-1] == "/":
  1206. path = f"{orig_path}{join_path}"
  1207. else:
  1208. # …
  1209. # and relativizing ".."
  1210. # parts[0] is / for absolute urls,
  1211. # this join will add a double slash there
  1212. path = "/".join([*self.parts[:-1], ""]) + join_path
  1213. # which has to be removed
  1214. if orig_path[0] == "/":
  1215. path = path[1:]
  1216. path = normalize_path(path) if "." in path else path
  1217. else:
  1218. path = orig_path
  1219. return from_parts(
  1220. scheme,
  1221. self._netloc,
  1222. path,
  1223. url._query if join_path or url._query else self._query,
  1224. url._fragment if join_path or url._fragment else self._fragment,
  1225. )
  1226. def joinpath(self, *other: str, encoded: bool = False) -> "URL":
  1227. """Return a new URL with the elements in other appended to the path."""
  1228. return self._make_child(other, encoded=encoded)
  1229. def human_repr(self) -> str:
  1230. """Return decoded human readable string for URL representation."""
  1231. user = human_quote(self.user, "#/:?@[]")
  1232. password = human_quote(self.password, "#/:?@[]")
  1233. if (host := self.host) and ":" in host:
  1234. host = f"[{host}]"
  1235. path = human_quote(self.path, "#?")
  1236. if TYPE_CHECKING:
  1237. assert path is not None
  1238. query_string = "&".join(
  1239. "{}={}".format(human_quote(k, "#&+;="), human_quote(v, "#&+;="))
  1240. for k, v in self.query.items()
  1241. )
  1242. fragment = human_quote(self.fragment, "")
  1243. if TYPE_CHECKING:
  1244. assert fragment is not None
  1245. netloc = make_netloc(user, password, host, self.explicit_port)
  1246. return unsplit_result(self._scheme, netloc, path, query_string, fragment)
  1247. _DEFAULT_IDNA_SIZE = 256
  1248. _DEFAULT_ENCODE_SIZE = 512
  1249. @lru_cache(_DEFAULT_IDNA_SIZE)
  1250. def _idna_decode(raw: str) -> str:
  1251. try:
  1252. return idna.decode(raw.encode("ascii"))
  1253. except UnicodeError: # e.g. '::1'
  1254. return raw.encode("ascii").decode("idna")
  1255. @lru_cache(_DEFAULT_IDNA_SIZE)
  1256. def _idna_encode(host: str) -> str:
  1257. try:
  1258. return idna.encode(host, uts46=True).decode("ascii")
  1259. except UnicodeError:
  1260. return host.encode("idna").decode("ascii")
  1261. @lru_cache(_DEFAULT_ENCODE_SIZE)
  1262. def _encode_host(host: str, validate_host: bool) -> str:
  1263. """Encode host part of URL."""
  1264. # If the host ends with a digit or contains a colon, its likely
  1265. # an IP address.
  1266. if host and (host[-1].isdigit() or ":" in host):
  1267. raw_ip, sep, zone = host.partition("%")
  1268. # If it looks like an IP, we check with _ip_compressed_version
  1269. # and fall-through if its not an IP address. This is a performance
  1270. # optimization to avoid parsing IP addresses as much as possible
  1271. # because it is orders of magnitude slower than almost any other
  1272. # operation this library does.
  1273. # Might be an IP address, check it
  1274. #
  1275. # IP Addresses can look like:
  1276. # https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
  1277. # - 127.0.0.1 (last character is a digit)
  1278. # - 2001:db8::ff00:42:8329 (contains a colon)
  1279. # - 2001:db8::ff00:42:8329%eth0 (contains a colon)
  1280. # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should
  1281. # have been removed before it gets here)
  1282. # Rare IP Address formats are not supported per:
  1283. # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4
  1284. #
  1285. # IP parsing is slow, so its wrapped in an LRU
  1286. try:
  1287. ip = ip_address(raw_ip)
  1288. except ValueError:
  1289. pass
  1290. else:
  1291. # These checks should not happen in the
  1292. # LRU to keep the cache size small
  1293. host = ip.compressed
  1294. if ip.version == 6:
  1295. return f"[{host}%{zone}]" if sep else f"[{host}]"
  1296. return f"{host}%{zone}" if sep else host
  1297. # IDNA encoding is slow, skip it for ASCII-only strings
  1298. if host.isascii():
  1299. # Check for invalid characters explicitly; _idna_encode() does this
  1300. # for non-ascii host names.
  1301. host = host.lower()
  1302. if validate_host and (invalid := NOT_REG_NAME.search(host)):
  1303. value, pos, extra = invalid.group(), invalid.start(), ""
  1304. if value == "@" or (value == ":" and "@" in host[pos:]):
  1305. # this looks like an authority string
  1306. extra = (
  1307. ", if the value includes a username or password, "
  1308. "use 'authority' instead of 'host'"
  1309. )
  1310. raise ValueError(
  1311. f"Host {host!r} cannot contain {value!r} (at position {pos}){extra}"
  1312. ) from None
  1313. return host
  1314. return _idna_encode(host)
  1315. @rewrite_module
  1316. def cache_clear() -> None:
  1317. """Clear all LRU caches."""
  1318. _idna_encode.cache_clear()
  1319. _idna_decode.cache_clear()
  1320. _encode_host.cache_clear()
  1321. @rewrite_module
  1322. def cache_info() -> CacheInfo:
  1323. """Report cache statistics."""
  1324. return {
  1325. "idna_encode": _idna_encode.cache_info(),
  1326. "idna_decode": _idna_decode.cache_info(),
  1327. "ip_address": _encode_host.cache_info(),
  1328. "host_validate": _encode_host.cache_info(),
  1329. "encode_host": _encode_host.cache_info(),
  1330. }
  1331. @rewrite_module
  1332. def cache_configure(
  1333. *,
  1334. idna_encode_size: Union[int, None] = _DEFAULT_IDNA_SIZE,
  1335. idna_decode_size: Union[int, None] = _DEFAULT_IDNA_SIZE,
  1336. ip_address_size: Union[int, None, UndefinedType] = UNDEFINED,
  1337. host_validate_size: Union[int, None, UndefinedType] = UNDEFINED,
  1338. encode_host_size: Union[int, None, UndefinedType] = UNDEFINED,
  1339. ) -> None:
  1340. """Configure LRU cache sizes."""
  1341. global _idna_decode, _idna_encode, _encode_host
  1342. # ip_address_size, host_validate_size are no longer
  1343. # used, but are kept for backwards compatibility.
  1344. if ip_address_size is not UNDEFINED or host_validate_size is not UNDEFINED:
  1345. warnings.warn(
  1346. "cache_configure() no longer accepts the "
  1347. "ip_address_size or host_validate_size arguments, "
  1348. "they are used to set the encode_host_size instead "
  1349. "and will be removed in the future",
  1350. DeprecationWarning,
  1351. stacklevel=2,
  1352. )
  1353. if encode_host_size is not None:
  1354. for size in (ip_address_size, host_validate_size):
  1355. if size is None:
  1356. encode_host_size = None
  1357. elif encode_host_size is UNDEFINED:
  1358. if size is not UNDEFINED:
  1359. encode_host_size = size
  1360. elif size is not UNDEFINED:
  1361. if TYPE_CHECKING:
  1362. assert isinstance(size, int)
  1363. assert isinstance(encode_host_size, int)
  1364. encode_host_size = max(size, encode_host_size)
  1365. if encode_host_size is UNDEFINED:
  1366. encode_host_size = _DEFAULT_ENCODE_SIZE
  1367. _encode_host = lru_cache(encode_host_size)(_encode_host.__wrapped__)
  1368. _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__)
  1369. _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__)