connection.py 65 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822
  1. import copy
  2. import os
  3. import socket
  4. import sys
  5. import threading
  6. import time
  7. import weakref
  8. from abc import abstractmethod
  9. from itertools import chain
  10. from queue import Empty, Full, LifoQueue
  11. from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union
  12. from urllib.parse import parse_qs, unquote, urlparse
  13. from redis.cache import (
  14. CacheEntry,
  15. CacheEntryStatus,
  16. CacheFactory,
  17. CacheFactoryInterface,
  18. CacheInterface,
  19. CacheKey,
  20. )
  21. from ._parsers import Encoder, _HiredisParser, _RESP2Parser, _RESP3Parser
  22. from .auth.token import TokenInterface
  23. from .backoff import NoBackoff
  24. from .credentials import CredentialProvider, UsernamePasswordCredentialProvider
  25. from .event import AfterConnectionReleasedEvent, EventDispatcher
  26. from .exceptions import (
  27. AuthenticationError,
  28. AuthenticationWrongNumberOfArgsError,
  29. ChildDeadlockedError,
  30. ConnectionError,
  31. DataError,
  32. RedisError,
  33. ResponseError,
  34. TimeoutError,
  35. )
  36. from .retry import Retry
  37. from .utils import (
  38. CRYPTOGRAPHY_AVAILABLE,
  39. HIREDIS_AVAILABLE,
  40. SSL_AVAILABLE,
  41. compare_versions,
  42. deprecated_args,
  43. ensure_string,
  44. format_error_message,
  45. get_lib_version,
  46. str_if_bytes,
  47. )
  48. if SSL_AVAILABLE:
  49. import ssl
  50. else:
  51. ssl = None
  52. if HIREDIS_AVAILABLE:
  53. import hiredis
  54. SYM_STAR = b"*"
  55. SYM_DOLLAR = b"$"
  56. SYM_CRLF = b"\r\n"
  57. SYM_EMPTY = b""
  58. DEFAULT_RESP_VERSION = 2
  59. SENTINEL = object()
  60. DefaultParser: Type[Union[_RESP2Parser, _RESP3Parser, _HiredisParser]]
  61. if HIREDIS_AVAILABLE:
  62. DefaultParser = _HiredisParser
  63. else:
  64. DefaultParser = _RESP2Parser
  65. class HiredisRespSerializer:
  66. def pack(self, *args: List):
  67. """Pack a series of arguments into the Redis protocol"""
  68. output = []
  69. if isinstance(args[0], str):
  70. args = tuple(args[0].encode().split()) + args[1:]
  71. elif b" " in args[0]:
  72. args = tuple(args[0].split()) + args[1:]
  73. try:
  74. output.append(hiredis.pack_command(args))
  75. except TypeError:
  76. _, value, traceback = sys.exc_info()
  77. raise DataError(value).with_traceback(traceback)
  78. return output
  79. class PythonRespSerializer:
  80. def __init__(self, buffer_cutoff, encode) -> None:
  81. self._buffer_cutoff = buffer_cutoff
  82. self.encode = encode
  83. def pack(self, *args):
  84. """Pack a series of arguments into the Redis protocol"""
  85. output = []
  86. # the client might have included 1 or more literal arguments in
  87. # the command name, e.g., 'CONFIG GET'. The Redis server expects these
  88. # arguments to be sent separately, so split the first argument
  89. # manually. These arguments should be bytestrings so that they are
  90. # not encoded.
  91. if isinstance(args[0], str):
  92. args = tuple(args[0].encode().split()) + args[1:]
  93. elif b" " in args[0]:
  94. args = tuple(args[0].split()) + args[1:]
  95. buff = SYM_EMPTY.join((SYM_STAR, str(len(args)).encode(), SYM_CRLF))
  96. buffer_cutoff = self._buffer_cutoff
  97. for arg in map(self.encode, args):
  98. # to avoid large string mallocs, chunk the command into the
  99. # output list if we're sending large values or memoryviews
  100. arg_length = len(arg)
  101. if (
  102. len(buff) > buffer_cutoff
  103. or arg_length > buffer_cutoff
  104. or isinstance(arg, memoryview)
  105. ):
  106. buff = SYM_EMPTY.join(
  107. (buff, SYM_DOLLAR, str(arg_length).encode(), SYM_CRLF)
  108. )
  109. output.append(buff)
  110. output.append(arg)
  111. buff = SYM_CRLF
  112. else:
  113. buff = SYM_EMPTY.join(
  114. (
  115. buff,
  116. SYM_DOLLAR,
  117. str(arg_length).encode(),
  118. SYM_CRLF,
  119. arg,
  120. SYM_CRLF,
  121. )
  122. )
  123. output.append(buff)
  124. return output
  125. class ConnectionInterface:
  126. @abstractmethod
  127. def repr_pieces(self):
  128. pass
  129. @abstractmethod
  130. def register_connect_callback(self, callback):
  131. pass
  132. @abstractmethod
  133. def deregister_connect_callback(self, callback):
  134. pass
  135. @abstractmethod
  136. def set_parser(self, parser_class):
  137. pass
  138. @abstractmethod
  139. def get_protocol(self):
  140. pass
  141. @abstractmethod
  142. def connect(self):
  143. pass
  144. @abstractmethod
  145. def on_connect(self):
  146. pass
  147. @abstractmethod
  148. def disconnect(self, *args):
  149. pass
  150. @abstractmethod
  151. def check_health(self):
  152. pass
  153. @abstractmethod
  154. def send_packed_command(self, command, check_health=True):
  155. pass
  156. @abstractmethod
  157. def send_command(self, *args, **kwargs):
  158. pass
  159. @abstractmethod
  160. def can_read(self, timeout=0):
  161. pass
  162. @abstractmethod
  163. def read_response(
  164. self,
  165. disable_decoding=False,
  166. *,
  167. disconnect_on_error=True,
  168. push_request=False,
  169. ):
  170. pass
  171. @abstractmethod
  172. def pack_command(self, *args):
  173. pass
  174. @abstractmethod
  175. def pack_commands(self, commands):
  176. pass
  177. @property
  178. @abstractmethod
  179. def handshake_metadata(self) -> Union[Dict[bytes, bytes], Dict[str, str]]:
  180. pass
  181. @abstractmethod
  182. def set_re_auth_token(self, token: TokenInterface):
  183. pass
  184. @abstractmethod
  185. def re_auth(self):
  186. pass
  187. class AbstractConnection(ConnectionInterface):
  188. "Manages communication to and from a Redis server"
  189. def __init__(
  190. self,
  191. db: int = 0,
  192. password: Optional[str] = None,
  193. socket_timeout: Optional[float] = None,
  194. socket_connect_timeout: Optional[float] = None,
  195. retry_on_timeout: bool = False,
  196. retry_on_error=SENTINEL,
  197. encoding: str = "utf-8",
  198. encoding_errors: str = "strict",
  199. decode_responses: bool = False,
  200. parser_class=DefaultParser,
  201. socket_read_size: int = 65536,
  202. health_check_interval: int = 0,
  203. client_name: Optional[str] = None,
  204. lib_name: Optional[str] = "redis-py",
  205. lib_version: Optional[str] = get_lib_version(),
  206. username: Optional[str] = None,
  207. retry: Union[Any, None] = None,
  208. redis_connect_func: Optional[Callable[[], None]] = None,
  209. credential_provider: Optional[CredentialProvider] = None,
  210. protocol: Optional[int] = 2,
  211. command_packer: Optional[Callable[[], None]] = None,
  212. event_dispatcher: Optional[EventDispatcher] = None,
  213. ):
  214. """
  215. Initialize a new Connection.
  216. To specify a retry policy for specific errors, first set
  217. `retry_on_error` to a list of the error/s to retry on, then set
  218. `retry` to a valid `Retry` object.
  219. To retry on TimeoutError, `retry_on_timeout` can also be set to `True`.
  220. """
  221. if (username or password) and credential_provider is not None:
  222. raise DataError(
  223. "'username' and 'password' cannot be passed along with 'credential_"
  224. "provider'. Please provide only one of the following arguments: \n"
  225. "1. 'password' and (optional) 'username'\n"
  226. "2. 'credential_provider'"
  227. )
  228. if event_dispatcher is None:
  229. self._event_dispatcher = EventDispatcher()
  230. else:
  231. self._event_dispatcher = event_dispatcher
  232. self.pid = os.getpid()
  233. self.db = db
  234. self.client_name = client_name
  235. self.lib_name = lib_name
  236. self.lib_version = lib_version
  237. self.credential_provider = credential_provider
  238. self.password = password
  239. self.username = username
  240. self.socket_timeout = socket_timeout
  241. if socket_connect_timeout is None:
  242. socket_connect_timeout = socket_timeout
  243. self.socket_connect_timeout = socket_connect_timeout
  244. self.retry_on_timeout = retry_on_timeout
  245. if retry_on_error is SENTINEL:
  246. retry_on_error = []
  247. if retry_on_timeout:
  248. # Add TimeoutError to the errors list to retry on
  249. retry_on_error.append(TimeoutError)
  250. self.retry_on_error = retry_on_error
  251. if retry or retry_on_error:
  252. if retry is None:
  253. self.retry = Retry(NoBackoff(), 1)
  254. else:
  255. # deep-copy the Retry object as it is mutable
  256. self.retry = copy.deepcopy(retry)
  257. # Update the retry's supported errors with the specified errors
  258. self.retry.update_supported_errors(retry_on_error)
  259. else:
  260. self.retry = Retry(NoBackoff(), 0)
  261. self.health_check_interval = health_check_interval
  262. self.next_health_check = 0
  263. self.redis_connect_func = redis_connect_func
  264. self.encoder = Encoder(encoding, encoding_errors, decode_responses)
  265. self.handshake_metadata = None
  266. self._sock = None
  267. self._socket_read_size = socket_read_size
  268. self.set_parser(parser_class)
  269. self._connect_callbacks = []
  270. self._buffer_cutoff = 6000
  271. self._re_auth_token: Optional[TokenInterface] = None
  272. try:
  273. p = int(protocol)
  274. except TypeError:
  275. p = DEFAULT_RESP_VERSION
  276. except ValueError:
  277. raise ConnectionError("protocol must be an integer")
  278. finally:
  279. if p < 2 or p > 3:
  280. raise ConnectionError("protocol must be either 2 or 3")
  281. # p = DEFAULT_RESP_VERSION
  282. self.protocol = p
  283. self._command_packer = self._construct_command_packer(command_packer)
  284. def __repr__(self):
  285. repr_args = ",".join([f"{k}={v}" for k, v in self.repr_pieces()])
  286. return f"<{self.__class__.__module__}.{self.__class__.__name__}({repr_args})>"
  287. @abstractmethod
  288. def repr_pieces(self):
  289. pass
  290. def __del__(self):
  291. try:
  292. self.disconnect()
  293. except Exception:
  294. pass
  295. def _construct_command_packer(self, packer):
  296. if packer is not None:
  297. return packer
  298. elif HIREDIS_AVAILABLE:
  299. return HiredisRespSerializer()
  300. else:
  301. return PythonRespSerializer(self._buffer_cutoff, self.encoder.encode)
  302. def register_connect_callback(self, callback):
  303. """
  304. Register a callback to be called when the connection is established either
  305. initially or reconnected. This allows listeners to issue commands that
  306. are ephemeral to the connection, for example pub/sub subscription or
  307. key tracking. The callback must be a _method_ and will be kept as
  308. a weak reference.
  309. """
  310. wm = weakref.WeakMethod(callback)
  311. if wm not in self._connect_callbacks:
  312. self._connect_callbacks.append(wm)
  313. def deregister_connect_callback(self, callback):
  314. """
  315. De-register a previously registered callback. It will no-longer receive
  316. notifications on connection events. Calling this is not required when the
  317. listener goes away, since the callbacks are kept as weak methods.
  318. """
  319. try:
  320. self._connect_callbacks.remove(weakref.WeakMethod(callback))
  321. except ValueError:
  322. pass
  323. def set_parser(self, parser_class):
  324. """
  325. Creates a new instance of parser_class with socket size:
  326. _socket_read_size and assigns it to the parser for the connection
  327. :param parser_class: The required parser class
  328. """
  329. self._parser = parser_class(socket_read_size=self._socket_read_size)
  330. def connect(self):
  331. "Connects to the Redis server if not already connected"
  332. self.connect_check_health(check_health=True)
  333. def connect_check_health(self, check_health: bool = True):
  334. if self._sock:
  335. return
  336. try:
  337. sock = self.retry.call_with_retry(
  338. lambda: self._connect(), lambda error: self.disconnect(error)
  339. )
  340. except socket.timeout:
  341. raise TimeoutError("Timeout connecting to server")
  342. except OSError as e:
  343. raise ConnectionError(self._error_message(e))
  344. self._sock = sock
  345. try:
  346. if self.redis_connect_func is None:
  347. # Use the default on_connect function
  348. self.on_connect_check_health(check_health=check_health)
  349. else:
  350. # Use the passed function redis_connect_func
  351. self.redis_connect_func(self)
  352. except RedisError:
  353. # clean up after any error in on_connect
  354. self.disconnect()
  355. raise
  356. # run any user callbacks. right now the only internal callback
  357. # is for pubsub channel/pattern resubscription
  358. # first, remove any dead weakrefs
  359. self._connect_callbacks = [ref for ref in self._connect_callbacks if ref()]
  360. for ref in self._connect_callbacks:
  361. callback = ref()
  362. if callback:
  363. callback(self)
  364. @abstractmethod
  365. def _connect(self):
  366. pass
  367. @abstractmethod
  368. def _host_error(self):
  369. pass
  370. def _error_message(self, exception):
  371. return format_error_message(self._host_error(), exception)
  372. def on_connect(self):
  373. self.on_connect_check_health(check_health=True)
  374. def on_connect_check_health(self, check_health: bool = True):
  375. "Initialize the connection, authenticate and select a database"
  376. self._parser.on_connect(self)
  377. parser = self._parser
  378. auth_args = None
  379. # if credential provider or username and/or password are set, authenticate
  380. if self.credential_provider or (self.username or self.password):
  381. cred_provider = (
  382. self.credential_provider
  383. or UsernamePasswordCredentialProvider(self.username, self.password)
  384. )
  385. auth_args = cred_provider.get_credentials()
  386. # if resp version is specified and we have auth args,
  387. # we need to send them via HELLO
  388. if auth_args and self.protocol not in [2, "2"]:
  389. if isinstance(self._parser, _RESP2Parser):
  390. self.set_parser(_RESP3Parser)
  391. # update cluster exception classes
  392. self._parser.EXCEPTION_CLASSES = parser.EXCEPTION_CLASSES
  393. self._parser.on_connect(self)
  394. if len(auth_args) == 1:
  395. auth_args = ["default", auth_args[0]]
  396. # avoid checking health here -- PING will fail if we try
  397. # to check the health prior to the AUTH
  398. self.send_command(
  399. "HELLO", self.protocol, "AUTH", *auth_args, check_health=False
  400. )
  401. self.handshake_metadata = self.read_response()
  402. # if response.get(b"proto") != self.protocol and response.get(
  403. # "proto"
  404. # ) != self.protocol:
  405. # raise ConnectionError("Invalid RESP version")
  406. elif auth_args:
  407. # avoid checking health here -- PING will fail if we try
  408. # to check the health prior to the AUTH
  409. self.send_command("AUTH", *auth_args, check_health=False)
  410. try:
  411. auth_response = self.read_response()
  412. except AuthenticationWrongNumberOfArgsError:
  413. # a username and password were specified but the Redis
  414. # server seems to be < 6.0.0 which expects a single password
  415. # arg. retry auth with just the password.
  416. # https://github.com/andymccurdy/redis-py/issues/1274
  417. self.send_command("AUTH", auth_args[-1], check_health=False)
  418. auth_response = self.read_response()
  419. if str_if_bytes(auth_response) != "OK":
  420. raise AuthenticationError("Invalid Username or Password")
  421. # if resp version is specified, switch to it
  422. elif self.protocol not in [2, "2"]:
  423. if isinstance(self._parser, _RESP2Parser):
  424. self.set_parser(_RESP3Parser)
  425. # update cluster exception classes
  426. self._parser.EXCEPTION_CLASSES = parser.EXCEPTION_CLASSES
  427. self._parser.on_connect(self)
  428. self.send_command("HELLO", self.protocol, check_health=check_health)
  429. self.handshake_metadata = self.read_response()
  430. if (
  431. self.handshake_metadata.get(b"proto") != self.protocol
  432. and self.handshake_metadata.get("proto") != self.protocol
  433. ):
  434. raise ConnectionError("Invalid RESP version")
  435. # if a client_name is given, set it
  436. if self.client_name:
  437. self.send_command(
  438. "CLIENT",
  439. "SETNAME",
  440. self.client_name,
  441. check_health=check_health,
  442. )
  443. if str_if_bytes(self.read_response()) != "OK":
  444. raise ConnectionError("Error setting client name")
  445. try:
  446. # set the library name and version
  447. if self.lib_name:
  448. self.send_command(
  449. "CLIENT",
  450. "SETINFO",
  451. "LIB-NAME",
  452. self.lib_name,
  453. check_health=check_health,
  454. )
  455. self.read_response()
  456. except ResponseError:
  457. pass
  458. try:
  459. if self.lib_version:
  460. self.send_command(
  461. "CLIENT",
  462. "SETINFO",
  463. "LIB-VER",
  464. self.lib_version,
  465. check_health=check_health,
  466. )
  467. self.read_response()
  468. except ResponseError:
  469. pass
  470. # if a database is specified, switch to it
  471. if self.db:
  472. self.send_command("SELECT", self.db, check_health=check_health)
  473. if str_if_bytes(self.read_response()) != "OK":
  474. raise ConnectionError("Invalid Database")
  475. def disconnect(self, *args):
  476. "Disconnects from the Redis server"
  477. self._parser.on_disconnect()
  478. conn_sock = self._sock
  479. self._sock = None
  480. if conn_sock is None:
  481. return
  482. if os.getpid() == self.pid:
  483. try:
  484. conn_sock.shutdown(socket.SHUT_RDWR)
  485. except (OSError, TypeError):
  486. pass
  487. try:
  488. conn_sock.close()
  489. except OSError:
  490. pass
  491. def _send_ping(self):
  492. """Send PING, expect PONG in return"""
  493. self.send_command("PING", check_health=False)
  494. if str_if_bytes(self.read_response()) != "PONG":
  495. raise ConnectionError("Bad response from PING health check")
  496. def _ping_failed(self, error):
  497. """Function to call when PING fails"""
  498. self.disconnect()
  499. def check_health(self):
  500. """Check the health of the connection with a PING/PONG"""
  501. if self.health_check_interval and time.monotonic() > self.next_health_check:
  502. self.retry.call_with_retry(self._send_ping, self._ping_failed)
  503. def send_packed_command(self, command, check_health=True):
  504. """Send an already packed command to the Redis server"""
  505. if not self._sock:
  506. self.connect_check_health(check_health=False)
  507. # guard against health check recursion
  508. if check_health:
  509. self.check_health()
  510. try:
  511. if isinstance(command, str):
  512. command = [command]
  513. for item in command:
  514. self._sock.sendall(item)
  515. except socket.timeout:
  516. self.disconnect()
  517. raise TimeoutError("Timeout writing to socket")
  518. except OSError as e:
  519. self.disconnect()
  520. if len(e.args) == 1:
  521. errno, errmsg = "UNKNOWN", e.args[0]
  522. else:
  523. errno = e.args[0]
  524. errmsg = e.args[1]
  525. raise ConnectionError(f"Error {errno} while writing to socket. {errmsg}.")
  526. except BaseException:
  527. # BaseExceptions can be raised when a socket send operation is not
  528. # finished, e.g. due to a timeout. Ideally, a caller could then re-try
  529. # to send un-sent data. However, the send_packed_command() API
  530. # does not support it so there is no point in keeping the connection open.
  531. self.disconnect()
  532. raise
  533. def send_command(self, *args, **kwargs):
  534. """Pack and send a command to the Redis server"""
  535. self.send_packed_command(
  536. self._command_packer.pack(*args),
  537. check_health=kwargs.get("check_health", True),
  538. )
  539. def can_read(self, timeout=0):
  540. """Poll the socket to see if there's data that can be read."""
  541. sock = self._sock
  542. if not sock:
  543. self.connect()
  544. host_error = self._host_error()
  545. try:
  546. return self._parser.can_read(timeout)
  547. except OSError as e:
  548. self.disconnect()
  549. raise ConnectionError(f"Error while reading from {host_error}: {e.args}")
  550. def read_response(
  551. self,
  552. disable_decoding=False,
  553. *,
  554. disconnect_on_error=True,
  555. push_request=False,
  556. ):
  557. """Read the response from a previously sent command"""
  558. host_error = self._host_error()
  559. try:
  560. if self.protocol in ["3", 3]:
  561. response = self._parser.read_response(
  562. disable_decoding=disable_decoding, push_request=push_request
  563. )
  564. else:
  565. response = self._parser.read_response(disable_decoding=disable_decoding)
  566. except socket.timeout:
  567. if disconnect_on_error:
  568. self.disconnect()
  569. raise TimeoutError(f"Timeout reading from {host_error}")
  570. except OSError as e:
  571. if disconnect_on_error:
  572. self.disconnect()
  573. raise ConnectionError(f"Error while reading from {host_error} : {e.args}")
  574. except BaseException:
  575. # Also by default close in case of BaseException. A lot of code
  576. # relies on this behaviour when doing Command/Response pairs.
  577. # See #1128.
  578. if disconnect_on_error:
  579. self.disconnect()
  580. raise
  581. if self.health_check_interval:
  582. self.next_health_check = time.monotonic() + self.health_check_interval
  583. if isinstance(response, ResponseError):
  584. try:
  585. raise response
  586. finally:
  587. del response # avoid creating ref cycles
  588. return response
  589. def pack_command(self, *args):
  590. """Pack a series of arguments into the Redis protocol"""
  591. return self._command_packer.pack(*args)
  592. def pack_commands(self, commands):
  593. """Pack multiple commands into the Redis protocol"""
  594. output = []
  595. pieces = []
  596. buffer_length = 0
  597. buffer_cutoff = self._buffer_cutoff
  598. for cmd in commands:
  599. for chunk in self._command_packer.pack(*cmd):
  600. chunklen = len(chunk)
  601. if (
  602. buffer_length > buffer_cutoff
  603. or chunklen > buffer_cutoff
  604. or isinstance(chunk, memoryview)
  605. ):
  606. if pieces:
  607. output.append(SYM_EMPTY.join(pieces))
  608. buffer_length = 0
  609. pieces = []
  610. if chunklen > buffer_cutoff or isinstance(chunk, memoryview):
  611. output.append(chunk)
  612. else:
  613. pieces.append(chunk)
  614. buffer_length += chunklen
  615. if pieces:
  616. output.append(SYM_EMPTY.join(pieces))
  617. return output
  618. def get_protocol(self) -> Union[int, str]:
  619. return self.protocol
  620. @property
  621. def handshake_metadata(self) -> Union[Dict[bytes, bytes], Dict[str, str]]:
  622. return self._handshake_metadata
  623. @handshake_metadata.setter
  624. def handshake_metadata(self, value: Union[Dict[bytes, bytes], Dict[str, str]]):
  625. self._handshake_metadata = value
  626. def set_re_auth_token(self, token: TokenInterface):
  627. self._re_auth_token = token
  628. def re_auth(self):
  629. if self._re_auth_token is not None:
  630. self.send_command(
  631. "AUTH",
  632. self._re_auth_token.try_get("oid"),
  633. self._re_auth_token.get_value(),
  634. )
  635. self.read_response()
  636. self._re_auth_token = None
  637. class Connection(AbstractConnection):
  638. "Manages TCP communication to and from a Redis server"
  639. def __init__(
  640. self,
  641. host="localhost",
  642. port=6379,
  643. socket_keepalive=False,
  644. socket_keepalive_options=None,
  645. socket_type=0,
  646. **kwargs,
  647. ):
  648. self.host = host
  649. self.port = int(port)
  650. self.socket_keepalive = socket_keepalive
  651. self.socket_keepalive_options = socket_keepalive_options or {}
  652. self.socket_type = socket_type
  653. super().__init__(**kwargs)
  654. def repr_pieces(self):
  655. pieces = [("host", self.host), ("port", self.port), ("db", self.db)]
  656. if self.client_name:
  657. pieces.append(("client_name", self.client_name))
  658. return pieces
  659. def _connect(self):
  660. "Create a TCP socket connection"
  661. # we want to mimic what socket.create_connection does to support
  662. # ipv4/ipv6, but we want to set options prior to calling
  663. # socket.connect()
  664. err = None
  665. for res in socket.getaddrinfo(
  666. self.host, self.port, self.socket_type, socket.SOCK_STREAM
  667. ):
  668. family, socktype, proto, canonname, socket_address = res
  669. sock = None
  670. try:
  671. sock = socket.socket(family, socktype, proto)
  672. # TCP_NODELAY
  673. sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
  674. # TCP_KEEPALIVE
  675. if self.socket_keepalive:
  676. sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
  677. for k, v in self.socket_keepalive_options.items():
  678. sock.setsockopt(socket.IPPROTO_TCP, k, v)
  679. # set the socket_connect_timeout before we connect
  680. sock.settimeout(self.socket_connect_timeout)
  681. # connect
  682. sock.connect(socket_address)
  683. # set the socket_timeout now that we're connected
  684. sock.settimeout(self.socket_timeout)
  685. return sock
  686. except OSError as _:
  687. err = _
  688. if sock is not None:
  689. try:
  690. sock.shutdown(socket.SHUT_RDWR) # ensure a clean close
  691. except OSError:
  692. pass
  693. sock.close()
  694. if err is not None:
  695. raise err
  696. raise OSError("socket.getaddrinfo returned an empty list")
  697. def _host_error(self):
  698. return f"{self.host}:{self.port}"
  699. class CacheProxyConnection(ConnectionInterface):
  700. DUMMY_CACHE_VALUE = b"foo"
  701. MIN_ALLOWED_VERSION = "7.4.0"
  702. DEFAULT_SERVER_NAME = "redis"
  703. def __init__(
  704. self,
  705. conn: ConnectionInterface,
  706. cache: CacheInterface,
  707. pool_lock: threading.Lock,
  708. ):
  709. self.pid = os.getpid()
  710. self._conn = conn
  711. self.retry = self._conn.retry
  712. self.host = self._conn.host
  713. self.port = self._conn.port
  714. self.credential_provider = conn.credential_provider
  715. self._pool_lock = pool_lock
  716. self._cache = cache
  717. self._cache_lock = threading.RLock()
  718. self._current_command_cache_key = None
  719. self._current_options = None
  720. self.register_connect_callback(self._enable_tracking_callback)
  721. def repr_pieces(self):
  722. return self._conn.repr_pieces()
  723. def register_connect_callback(self, callback):
  724. self._conn.register_connect_callback(callback)
  725. def deregister_connect_callback(self, callback):
  726. self._conn.deregister_connect_callback(callback)
  727. def set_parser(self, parser_class):
  728. self._conn.set_parser(parser_class)
  729. def connect(self):
  730. self._conn.connect()
  731. server_name = self._conn.handshake_metadata.get(b"server", None)
  732. if server_name is None:
  733. server_name = self._conn.handshake_metadata.get("server", None)
  734. server_ver = self._conn.handshake_metadata.get(b"version", None)
  735. if server_ver is None:
  736. server_ver = self._conn.handshake_metadata.get("version", None)
  737. if server_ver is None or server_ver is None:
  738. raise ConnectionError("Cannot retrieve information about server version")
  739. server_ver = ensure_string(server_ver)
  740. server_name = ensure_string(server_name)
  741. if (
  742. server_name != self.DEFAULT_SERVER_NAME
  743. or compare_versions(server_ver, self.MIN_ALLOWED_VERSION) == 1
  744. ):
  745. raise ConnectionError(
  746. "To maximize compatibility with all Redis products, client-side caching is supported by Redis 7.4 or later" # noqa: E501
  747. )
  748. def on_connect(self):
  749. self._conn.on_connect()
  750. def disconnect(self, *args):
  751. with self._cache_lock:
  752. self._cache.flush()
  753. self._conn.disconnect(*args)
  754. def check_health(self):
  755. self._conn.check_health()
  756. def send_packed_command(self, command, check_health=True):
  757. # TODO: Investigate if it's possible to unpack command
  758. # or extract keys from packed command
  759. self._conn.send_packed_command(command)
  760. def send_command(self, *args, **kwargs):
  761. self._process_pending_invalidations()
  762. with self._cache_lock:
  763. # Command is write command or not allowed
  764. # to be cached.
  765. if not self._cache.is_cachable(CacheKey(command=args[0], redis_keys=())):
  766. self._current_command_cache_key = None
  767. self._conn.send_command(*args, **kwargs)
  768. return
  769. if kwargs.get("keys") is None:
  770. raise ValueError("Cannot create cache key.")
  771. # Creates cache key.
  772. self._current_command_cache_key = CacheKey(
  773. command=args[0], redis_keys=tuple(kwargs.get("keys"))
  774. )
  775. with self._cache_lock:
  776. # We have to trigger invalidation processing in case if
  777. # it was cached by another connection to avoid
  778. # queueing invalidations in stale connections.
  779. if self._cache.get(self._current_command_cache_key):
  780. entry = self._cache.get(self._current_command_cache_key)
  781. if entry.connection_ref != self._conn:
  782. with self._pool_lock:
  783. while entry.connection_ref.can_read():
  784. entry.connection_ref.read_response(push_request=True)
  785. return
  786. # Set temporary entry value to prevent
  787. # race condition from another connection.
  788. self._cache.set(
  789. CacheEntry(
  790. cache_key=self._current_command_cache_key,
  791. cache_value=self.DUMMY_CACHE_VALUE,
  792. status=CacheEntryStatus.IN_PROGRESS,
  793. connection_ref=self._conn,
  794. )
  795. )
  796. # Send command over socket only if it's allowed
  797. # read-only command that not yet cached.
  798. self._conn.send_command(*args, **kwargs)
  799. def can_read(self, timeout=0):
  800. return self._conn.can_read(timeout)
  801. def read_response(
  802. self, disable_decoding=False, *, disconnect_on_error=True, push_request=False
  803. ):
  804. with self._cache_lock:
  805. # Check if command response exists in a cache and it's not in progress.
  806. if (
  807. self._current_command_cache_key is not None
  808. and self._cache.get(self._current_command_cache_key) is not None
  809. and self._cache.get(self._current_command_cache_key).status
  810. != CacheEntryStatus.IN_PROGRESS
  811. ):
  812. res = copy.deepcopy(
  813. self._cache.get(self._current_command_cache_key).cache_value
  814. )
  815. self._current_command_cache_key = None
  816. return res
  817. response = self._conn.read_response(
  818. disable_decoding=disable_decoding,
  819. disconnect_on_error=disconnect_on_error,
  820. push_request=push_request,
  821. )
  822. with self._cache_lock:
  823. # Prevent not-allowed command from caching.
  824. if self._current_command_cache_key is None:
  825. return response
  826. # If response is None prevent from caching.
  827. if response is None:
  828. self._cache.delete_by_cache_keys([self._current_command_cache_key])
  829. return response
  830. cache_entry = self._cache.get(self._current_command_cache_key)
  831. # Cache only responses that still valid
  832. # and wasn't invalidated by another connection in meantime.
  833. if cache_entry is not None:
  834. cache_entry.status = CacheEntryStatus.VALID
  835. cache_entry.cache_value = response
  836. self._cache.set(cache_entry)
  837. self._current_command_cache_key = None
  838. return response
  839. def pack_command(self, *args):
  840. return self._conn.pack_command(*args)
  841. def pack_commands(self, commands):
  842. return self._conn.pack_commands(commands)
  843. @property
  844. def handshake_metadata(self) -> Union[Dict[bytes, bytes], Dict[str, str]]:
  845. return self._conn.handshake_metadata
  846. def _connect(self):
  847. self._conn._connect()
  848. def _host_error(self):
  849. self._conn._host_error()
  850. def _enable_tracking_callback(self, conn: ConnectionInterface) -> None:
  851. conn.send_command("CLIENT", "TRACKING", "ON")
  852. conn.read_response()
  853. conn._parser.set_invalidation_push_handler(self._on_invalidation_callback)
  854. def _process_pending_invalidations(self):
  855. while self.can_read():
  856. self._conn.read_response(push_request=True)
  857. def _on_invalidation_callback(self, data: List[Union[str, Optional[List[bytes]]]]):
  858. with self._cache_lock:
  859. # Flush cache when DB flushed on server-side
  860. if data[1] is None:
  861. self._cache.flush()
  862. else:
  863. self._cache.delete_by_redis_keys(data[1])
  864. def get_protocol(self):
  865. return self._conn.get_protocol()
  866. def set_re_auth_token(self, token: TokenInterface):
  867. self._conn.set_re_auth_token(token)
  868. def re_auth(self):
  869. self._conn.re_auth()
  870. class SSLConnection(Connection):
  871. """Manages SSL connections to and from the Redis server(s).
  872. This class extends the Connection class, adding SSL functionality, and making
  873. use of ssl.SSLContext (https://docs.python.org/3/library/ssl.html#ssl.SSLContext)
  874. """ # noqa
  875. def __init__(
  876. self,
  877. ssl_keyfile=None,
  878. ssl_certfile=None,
  879. ssl_cert_reqs="required",
  880. ssl_ca_certs=None,
  881. ssl_ca_data=None,
  882. ssl_check_hostname=True,
  883. ssl_ca_path=None,
  884. ssl_password=None,
  885. ssl_validate_ocsp=False,
  886. ssl_validate_ocsp_stapled=False,
  887. ssl_ocsp_context=None,
  888. ssl_ocsp_expected_cert=None,
  889. ssl_min_version=None,
  890. ssl_ciphers=None,
  891. **kwargs,
  892. ):
  893. """Constructor
  894. Args:
  895. ssl_keyfile: Path to an ssl private key. Defaults to None.
  896. ssl_certfile: Path to an ssl certificate. Defaults to None.
  897. ssl_cert_reqs: The string value for the SSLContext.verify_mode (none, optional, required), or an ssl.VerifyMode. Defaults to "required".
  898. ssl_ca_certs: The path to a file of concatenated CA certificates in PEM format. Defaults to None.
  899. ssl_ca_data: Either an ASCII string of one or more PEM-encoded certificates or a bytes-like object of DER-encoded certificates.
  900. ssl_check_hostname: If set, match the hostname during the SSL handshake. Defaults to False.
  901. ssl_ca_path: The path to a directory containing several CA certificates in PEM format. Defaults to None.
  902. ssl_password: Password for unlocking an encrypted private key. Defaults to None.
  903. ssl_validate_ocsp: If set, perform a full ocsp validation (i.e not a stapled verification)
  904. ssl_validate_ocsp_stapled: If set, perform a validation on a stapled ocsp response
  905. ssl_ocsp_context: A fully initialized OpenSSL.SSL.Context object to be used in verifying the ssl_ocsp_expected_cert
  906. ssl_ocsp_expected_cert: A PEM armoured string containing the expected certificate to be returned from the ocsp verification service.
  907. ssl_min_version: The lowest supported SSL version. It affects the supported SSL versions of the SSLContext. None leaves the default provided by ssl module.
  908. ssl_ciphers: A string listing the ciphers that are allowed to be used. Defaults to None, which means that the default ciphers are used. See https://docs.python.org/3/library/ssl.html#ssl.SSLContext.set_ciphers for more information.
  909. Raises:
  910. RedisError
  911. """ # noqa
  912. if not SSL_AVAILABLE:
  913. raise RedisError("Python wasn't built with SSL support")
  914. self.keyfile = ssl_keyfile
  915. self.certfile = ssl_certfile
  916. if ssl_cert_reqs is None:
  917. ssl_cert_reqs = ssl.CERT_NONE
  918. elif isinstance(ssl_cert_reqs, str):
  919. CERT_REQS = { # noqa: N806
  920. "none": ssl.CERT_NONE,
  921. "optional": ssl.CERT_OPTIONAL,
  922. "required": ssl.CERT_REQUIRED,
  923. }
  924. if ssl_cert_reqs not in CERT_REQS:
  925. raise RedisError(
  926. f"Invalid SSL Certificate Requirements Flag: {ssl_cert_reqs}"
  927. )
  928. ssl_cert_reqs = CERT_REQS[ssl_cert_reqs]
  929. self.cert_reqs = ssl_cert_reqs
  930. self.ca_certs = ssl_ca_certs
  931. self.ca_data = ssl_ca_data
  932. self.ca_path = ssl_ca_path
  933. self.check_hostname = (
  934. ssl_check_hostname if self.cert_reqs != ssl.CERT_NONE else False
  935. )
  936. self.certificate_password = ssl_password
  937. self.ssl_validate_ocsp = ssl_validate_ocsp
  938. self.ssl_validate_ocsp_stapled = ssl_validate_ocsp_stapled
  939. self.ssl_ocsp_context = ssl_ocsp_context
  940. self.ssl_ocsp_expected_cert = ssl_ocsp_expected_cert
  941. self.ssl_min_version = ssl_min_version
  942. self.ssl_ciphers = ssl_ciphers
  943. super().__init__(**kwargs)
  944. def _connect(self):
  945. """
  946. Wrap the socket with SSL support, handling potential errors.
  947. """
  948. sock = super()._connect()
  949. try:
  950. return self._wrap_socket_with_ssl(sock)
  951. except (OSError, RedisError):
  952. sock.close()
  953. raise
  954. def _wrap_socket_with_ssl(self, sock):
  955. """
  956. Wraps the socket with SSL support.
  957. Args:
  958. sock: The plain socket to wrap with SSL.
  959. Returns:
  960. An SSL wrapped socket.
  961. """
  962. context = ssl.create_default_context()
  963. context.check_hostname = self.check_hostname
  964. context.verify_mode = self.cert_reqs
  965. if self.certfile or self.keyfile:
  966. context.load_cert_chain(
  967. certfile=self.certfile,
  968. keyfile=self.keyfile,
  969. password=self.certificate_password,
  970. )
  971. if (
  972. self.ca_certs is not None
  973. or self.ca_path is not None
  974. or self.ca_data is not None
  975. ):
  976. context.load_verify_locations(
  977. cafile=self.ca_certs, capath=self.ca_path, cadata=self.ca_data
  978. )
  979. if self.ssl_min_version is not None:
  980. context.minimum_version = self.ssl_min_version
  981. if self.ssl_ciphers:
  982. context.set_ciphers(self.ssl_ciphers)
  983. if self.ssl_validate_ocsp is True and CRYPTOGRAPHY_AVAILABLE is False:
  984. raise RedisError("cryptography is not installed.")
  985. if self.ssl_validate_ocsp_stapled and self.ssl_validate_ocsp:
  986. raise RedisError(
  987. "Either an OCSP staple or pure OCSP connection must be validated "
  988. "- not both."
  989. )
  990. sslsock = context.wrap_socket(sock, server_hostname=self.host)
  991. # validation for the stapled case
  992. if self.ssl_validate_ocsp_stapled:
  993. import OpenSSL
  994. from .ocsp import ocsp_staple_verifier
  995. # if a context is provided use it - otherwise, a basic context
  996. if self.ssl_ocsp_context is None:
  997. staple_ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
  998. staple_ctx.use_certificate_file(self.certfile)
  999. staple_ctx.use_privatekey_file(self.keyfile)
  1000. else:
  1001. staple_ctx = self.ssl_ocsp_context
  1002. staple_ctx.set_ocsp_client_callback(
  1003. ocsp_staple_verifier, self.ssl_ocsp_expected_cert
  1004. )
  1005. # need another socket
  1006. con = OpenSSL.SSL.Connection(staple_ctx, socket.socket())
  1007. con.request_ocsp()
  1008. con.connect((self.host, self.port))
  1009. con.do_handshake()
  1010. con.shutdown()
  1011. return sslsock
  1012. # pure ocsp validation
  1013. if self.ssl_validate_ocsp is True and CRYPTOGRAPHY_AVAILABLE:
  1014. from .ocsp import OCSPVerifier
  1015. o = OCSPVerifier(sslsock, self.host, self.port, self.ca_certs)
  1016. if o.is_valid():
  1017. return sslsock
  1018. else:
  1019. raise ConnectionError("ocsp validation error")
  1020. return sslsock
  1021. class UnixDomainSocketConnection(AbstractConnection):
  1022. "Manages UDS communication to and from a Redis server"
  1023. def __init__(self, path="", socket_timeout=None, **kwargs):
  1024. super().__init__(**kwargs)
  1025. self.path = path
  1026. self.socket_timeout = socket_timeout
  1027. def repr_pieces(self):
  1028. pieces = [("path", self.path), ("db", self.db)]
  1029. if self.client_name:
  1030. pieces.append(("client_name", self.client_name))
  1031. return pieces
  1032. def _connect(self):
  1033. "Create a Unix domain socket connection"
  1034. sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
  1035. sock.settimeout(self.socket_connect_timeout)
  1036. try:
  1037. sock.connect(self.path)
  1038. except OSError:
  1039. # Prevent ResourceWarnings for unclosed sockets.
  1040. try:
  1041. sock.shutdown(socket.SHUT_RDWR) # ensure a clean close
  1042. except OSError:
  1043. pass
  1044. sock.close()
  1045. raise
  1046. sock.settimeout(self.socket_timeout)
  1047. return sock
  1048. def _host_error(self):
  1049. return self.path
  1050. FALSE_STRINGS = ("0", "F", "FALSE", "N", "NO")
  1051. def to_bool(value):
  1052. if value is None or value == "":
  1053. return None
  1054. if isinstance(value, str) and value.upper() in FALSE_STRINGS:
  1055. return False
  1056. return bool(value)
  1057. URL_QUERY_ARGUMENT_PARSERS = {
  1058. "db": int,
  1059. "socket_timeout": float,
  1060. "socket_connect_timeout": float,
  1061. "socket_keepalive": to_bool,
  1062. "retry_on_timeout": to_bool,
  1063. "retry_on_error": list,
  1064. "max_connections": int,
  1065. "health_check_interval": int,
  1066. "ssl_check_hostname": to_bool,
  1067. "timeout": float,
  1068. }
  1069. def parse_url(url):
  1070. if not (
  1071. url.startswith("redis://")
  1072. or url.startswith("rediss://")
  1073. or url.startswith("unix://")
  1074. ):
  1075. raise ValueError(
  1076. "Redis URL must specify one of the following "
  1077. "schemes (redis://, rediss://, unix://)"
  1078. )
  1079. url = urlparse(url)
  1080. kwargs = {}
  1081. for name, value in parse_qs(url.query).items():
  1082. if value and len(value) > 0:
  1083. value = unquote(value[0])
  1084. parser = URL_QUERY_ARGUMENT_PARSERS.get(name)
  1085. if parser:
  1086. try:
  1087. kwargs[name] = parser(value)
  1088. except (TypeError, ValueError):
  1089. raise ValueError(f"Invalid value for '{name}' in connection URL.")
  1090. else:
  1091. kwargs[name] = value
  1092. if url.username:
  1093. kwargs["username"] = unquote(url.username)
  1094. if url.password:
  1095. kwargs["password"] = unquote(url.password)
  1096. # We only support redis://, rediss:// and unix:// schemes.
  1097. if url.scheme == "unix":
  1098. if url.path:
  1099. kwargs["path"] = unquote(url.path)
  1100. kwargs["connection_class"] = UnixDomainSocketConnection
  1101. else: # implied: url.scheme in ("redis", "rediss"):
  1102. if url.hostname:
  1103. kwargs["host"] = unquote(url.hostname)
  1104. if url.port:
  1105. kwargs["port"] = int(url.port)
  1106. # If there's a path argument, use it as the db argument if a
  1107. # querystring value wasn't specified
  1108. if url.path and "db" not in kwargs:
  1109. try:
  1110. kwargs["db"] = int(unquote(url.path).replace("/", ""))
  1111. except (AttributeError, ValueError):
  1112. pass
  1113. if url.scheme == "rediss":
  1114. kwargs["connection_class"] = SSLConnection
  1115. return kwargs
  1116. _CP = TypeVar("_CP", bound="ConnectionPool")
  1117. class ConnectionPool:
  1118. """
  1119. Create a connection pool. ``If max_connections`` is set, then this
  1120. object raises :py:class:`~redis.exceptions.ConnectionError` when the pool's
  1121. limit is reached.
  1122. By default, TCP connections are created unless ``connection_class``
  1123. is specified. Use class:`.UnixDomainSocketConnection` for
  1124. unix sockets.
  1125. Any additional keyword arguments are passed to the constructor of
  1126. ``connection_class``.
  1127. """
  1128. @classmethod
  1129. def from_url(cls: Type[_CP], url: str, **kwargs) -> _CP:
  1130. """
  1131. Return a connection pool configured from the given URL.
  1132. For example::
  1133. redis://[[username]:[password]]@localhost:6379/0
  1134. rediss://[[username]:[password]]@localhost:6379/0
  1135. unix://[username@]/path/to/socket.sock?db=0[&password=password]
  1136. Three URL schemes are supported:
  1137. - `redis://` creates a TCP socket connection. See more at:
  1138. <https://www.iana.org/assignments/uri-schemes/prov/redis>
  1139. - `rediss://` creates a SSL wrapped TCP socket connection. See more at:
  1140. <https://www.iana.org/assignments/uri-schemes/prov/rediss>
  1141. - ``unix://``: creates a Unix Domain Socket connection.
  1142. The username, password, hostname, path and all querystring values
  1143. are passed through urllib.parse.unquote in order to replace any
  1144. percent-encoded values with their corresponding characters.
  1145. There are several ways to specify a database number. The first value
  1146. found will be used:
  1147. 1. A ``db`` querystring option, e.g. redis://localhost?db=0
  1148. 2. If using the redis:// or rediss:// schemes, the path argument
  1149. of the url, e.g. redis://localhost/0
  1150. 3. A ``db`` keyword argument to this function.
  1151. If none of these options are specified, the default db=0 is used.
  1152. All querystring options are cast to their appropriate Python types.
  1153. Boolean arguments can be specified with string values "True"/"False"
  1154. or "Yes"/"No". Values that cannot be properly cast cause a
  1155. ``ValueError`` to be raised. Once parsed, the querystring arguments
  1156. and keyword arguments are passed to the ``ConnectionPool``'s
  1157. class initializer. In the case of conflicting arguments, querystring
  1158. arguments always win.
  1159. """
  1160. url_options = parse_url(url)
  1161. if "connection_class" in kwargs:
  1162. url_options["connection_class"] = kwargs["connection_class"]
  1163. kwargs.update(url_options)
  1164. return cls(**kwargs)
  1165. def __init__(
  1166. self,
  1167. connection_class=Connection,
  1168. max_connections: Optional[int] = None,
  1169. cache_factory: Optional[CacheFactoryInterface] = None,
  1170. **connection_kwargs,
  1171. ):
  1172. max_connections = max_connections or 2**31
  1173. if not isinstance(max_connections, int) or max_connections < 0:
  1174. raise ValueError('"max_connections" must be a positive integer')
  1175. self.connection_class = connection_class
  1176. self.connection_kwargs = connection_kwargs
  1177. self.max_connections = max_connections
  1178. self.cache = None
  1179. self._cache_factory = cache_factory
  1180. if connection_kwargs.get("cache_config") or connection_kwargs.get("cache"):
  1181. if connection_kwargs.get("protocol") not in [3, "3"]:
  1182. raise RedisError("Client caching is only supported with RESP version 3")
  1183. cache = self.connection_kwargs.get("cache")
  1184. if cache is not None:
  1185. if not isinstance(cache, CacheInterface):
  1186. raise ValueError("Cache must implement CacheInterface")
  1187. self.cache = cache
  1188. else:
  1189. if self._cache_factory is not None:
  1190. self.cache = self._cache_factory.get_cache()
  1191. else:
  1192. self.cache = CacheFactory(
  1193. self.connection_kwargs.get("cache_config")
  1194. ).get_cache()
  1195. connection_kwargs.pop("cache", None)
  1196. connection_kwargs.pop("cache_config", None)
  1197. self._event_dispatcher = self.connection_kwargs.get("event_dispatcher", None)
  1198. if self._event_dispatcher is None:
  1199. self._event_dispatcher = EventDispatcher()
  1200. # a lock to protect the critical section in _checkpid().
  1201. # this lock is acquired when the process id changes, such as
  1202. # after a fork. during this time, multiple threads in the child
  1203. # process could attempt to acquire this lock. the first thread
  1204. # to acquire the lock will reset the data structures and lock
  1205. # object of this pool. subsequent threads acquiring this lock
  1206. # will notice the first thread already did the work and simply
  1207. # release the lock.
  1208. self._fork_lock = threading.RLock()
  1209. if self.cache is None:
  1210. self._lock = threading.RLock()
  1211. else:
  1212. # TODO: To avoid breaking changes during the bug fix, we have to keep non-reentrant lock.
  1213. # TODO: Remove this before next major version (7.0.0)
  1214. self._lock = threading.Lock()
  1215. self.reset()
  1216. def __repr__(self) -> (str, str):
  1217. return (
  1218. f"<{type(self).__module__}.{type(self).__name__}"
  1219. f"({repr(self.connection_class(**self.connection_kwargs))})>"
  1220. )
  1221. def get_protocol(self):
  1222. """
  1223. Returns:
  1224. The RESP protocol version, or ``None`` if the protocol is not specified,
  1225. in which case the server default will be used.
  1226. """
  1227. return self.connection_kwargs.get("protocol", None)
  1228. def reset(self) -> None:
  1229. self._created_connections = 0
  1230. self._available_connections = []
  1231. self._in_use_connections = set()
  1232. # this must be the last operation in this method. while reset() is
  1233. # called when holding _fork_lock, other threads in this process
  1234. # can call _checkpid() which compares self.pid and os.getpid() without
  1235. # holding any lock (for performance reasons). keeping this assignment
  1236. # as the last operation ensures that those other threads will also
  1237. # notice a pid difference and block waiting for the first thread to
  1238. # release _fork_lock. when each of these threads eventually acquire
  1239. # _fork_lock, they will notice that another thread already called
  1240. # reset() and they will immediately release _fork_lock and continue on.
  1241. self.pid = os.getpid()
  1242. def _checkpid(self) -> None:
  1243. # _checkpid() attempts to keep ConnectionPool fork-safe on modern
  1244. # systems. this is called by all ConnectionPool methods that
  1245. # manipulate the pool's state such as get_connection() and release().
  1246. #
  1247. # _checkpid() determines whether the process has forked by comparing
  1248. # the current process id to the process id saved on the ConnectionPool
  1249. # instance. if these values are the same, _checkpid() simply returns.
  1250. #
  1251. # when the process ids differ, _checkpid() assumes that the process
  1252. # has forked and that we're now running in the child process. the child
  1253. # process cannot use the parent's file descriptors (e.g., sockets).
  1254. # therefore, when _checkpid() sees the process id change, it calls
  1255. # reset() in order to reinitialize the child's ConnectionPool. this
  1256. # will cause the child to make all new connection objects.
  1257. #
  1258. # _checkpid() is protected by self._fork_lock to ensure that multiple
  1259. # threads in the child process do not call reset() multiple times.
  1260. #
  1261. # there is an extremely small chance this could fail in the following
  1262. # scenario:
  1263. # 1. process A calls _checkpid() for the first time and acquires
  1264. # self._fork_lock.
  1265. # 2. while holding self._fork_lock, process A forks (the fork()
  1266. # could happen in a different thread owned by process A)
  1267. # 3. process B (the forked child process) inherits the
  1268. # ConnectionPool's state from the parent. that state includes
  1269. # a locked _fork_lock. process B will not be notified when
  1270. # process A releases the _fork_lock and will thus never be
  1271. # able to acquire the _fork_lock.
  1272. #
  1273. # to mitigate this possible deadlock, _checkpid() will only wait 5
  1274. # seconds to acquire _fork_lock. if _fork_lock cannot be acquired in
  1275. # that time it is assumed that the child is deadlocked and a
  1276. # redis.ChildDeadlockedError error is raised.
  1277. if self.pid != os.getpid():
  1278. acquired = self._fork_lock.acquire(timeout=5)
  1279. if not acquired:
  1280. raise ChildDeadlockedError
  1281. # reset() the instance for the new process if another thread
  1282. # hasn't already done so
  1283. try:
  1284. if self.pid != os.getpid():
  1285. self.reset()
  1286. finally:
  1287. self._fork_lock.release()
  1288. @deprecated_args(
  1289. args_to_warn=["*"],
  1290. reason="Use get_connection() without args instead",
  1291. version="5.3.0",
  1292. )
  1293. def get_connection(self, command_name=None, *keys, **options) -> "Connection":
  1294. "Get a connection from the pool"
  1295. self._checkpid()
  1296. with self._lock:
  1297. try:
  1298. connection = self._available_connections.pop()
  1299. except IndexError:
  1300. connection = self.make_connection()
  1301. self._in_use_connections.add(connection)
  1302. try:
  1303. # ensure this connection is connected to Redis
  1304. connection.connect()
  1305. # connections that the pool provides should be ready to send
  1306. # a command. if not, the connection was either returned to the
  1307. # pool before all data has been read or the socket has been
  1308. # closed. either way, reconnect and verify everything is good.
  1309. try:
  1310. if connection.can_read() and self.cache is None:
  1311. raise ConnectionError("Connection has data")
  1312. except (ConnectionError, TimeoutError, OSError):
  1313. connection.disconnect()
  1314. connection.connect()
  1315. if connection.can_read():
  1316. raise ConnectionError("Connection not ready")
  1317. except BaseException:
  1318. # release the connection back to the pool so that we don't
  1319. # leak it
  1320. self.release(connection)
  1321. raise
  1322. return connection
  1323. def get_encoder(self) -> Encoder:
  1324. "Return an encoder based on encoding settings"
  1325. kwargs = self.connection_kwargs
  1326. return Encoder(
  1327. encoding=kwargs.get("encoding", "utf-8"),
  1328. encoding_errors=kwargs.get("encoding_errors", "strict"),
  1329. decode_responses=kwargs.get("decode_responses", False),
  1330. )
  1331. def make_connection(self) -> "ConnectionInterface":
  1332. "Create a new connection"
  1333. if self._created_connections >= self.max_connections:
  1334. raise ConnectionError("Too many connections")
  1335. self._created_connections += 1
  1336. if self.cache is not None:
  1337. return CacheProxyConnection(
  1338. self.connection_class(**self.connection_kwargs), self.cache, self._lock
  1339. )
  1340. return self.connection_class(**self.connection_kwargs)
  1341. def release(self, connection: "Connection") -> None:
  1342. "Releases the connection back to the pool"
  1343. self._checkpid()
  1344. with self._lock:
  1345. try:
  1346. self._in_use_connections.remove(connection)
  1347. except KeyError:
  1348. # Gracefully fail when a connection is returned to this pool
  1349. # that the pool doesn't actually own
  1350. return
  1351. if self.owns_connection(connection):
  1352. self._available_connections.append(connection)
  1353. self._event_dispatcher.dispatch(
  1354. AfterConnectionReleasedEvent(connection)
  1355. )
  1356. else:
  1357. # Pool doesn't own this connection, do not add it back
  1358. # to the pool.
  1359. # The created connections count should not be changed,
  1360. # because the connection was not created by the pool.
  1361. connection.disconnect()
  1362. return
  1363. def owns_connection(self, connection: "Connection") -> int:
  1364. return connection.pid == self.pid
  1365. def disconnect(self, inuse_connections: bool = True) -> None:
  1366. """
  1367. Disconnects connections in the pool
  1368. If ``inuse_connections`` is True, disconnect connections that are
  1369. current in use, potentially by other threads. Otherwise only disconnect
  1370. connections that are idle in the pool.
  1371. """
  1372. self._checkpid()
  1373. with self._lock:
  1374. if inuse_connections:
  1375. connections = chain(
  1376. self._available_connections, self._in_use_connections
  1377. )
  1378. else:
  1379. connections = self._available_connections
  1380. for connection in connections:
  1381. connection.disconnect()
  1382. def close(self) -> None:
  1383. """Close the pool, disconnecting all connections"""
  1384. self.disconnect()
  1385. def set_retry(self, retry: Retry) -> None:
  1386. self.connection_kwargs.update({"retry": retry})
  1387. for conn in self._available_connections:
  1388. conn.retry = retry
  1389. for conn in self._in_use_connections:
  1390. conn.retry = retry
  1391. def re_auth_callback(self, token: TokenInterface):
  1392. with self._lock:
  1393. for conn in self._available_connections:
  1394. conn.retry.call_with_retry(
  1395. lambda: conn.send_command(
  1396. "AUTH", token.try_get("oid"), token.get_value()
  1397. ),
  1398. lambda error: self._mock(error),
  1399. )
  1400. conn.retry.call_with_retry(
  1401. lambda: conn.read_response(), lambda error: self._mock(error)
  1402. )
  1403. for conn in self._in_use_connections:
  1404. conn.set_re_auth_token(token)
  1405. async def _mock(self, error: RedisError):
  1406. """
  1407. Dummy functions, needs to be passed as error callback to retry object.
  1408. :param error:
  1409. :return:
  1410. """
  1411. pass
  1412. class BlockingConnectionPool(ConnectionPool):
  1413. """
  1414. Thread-safe blocking connection pool::
  1415. >>> from redis.client import Redis
  1416. >>> client = Redis(connection_pool=BlockingConnectionPool())
  1417. It performs the same function as the default
  1418. :py:class:`~redis.ConnectionPool` implementation, in that,
  1419. it maintains a pool of reusable connections that can be shared by
  1420. multiple redis clients (safely across threads if required).
  1421. The difference is that, in the event that a client tries to get a
  1422. connection from the pool when all of connections are in use, rather than
  1423. raising a :py:class:`~redis.ConnectionError` (as the default
  1424. :py:class:`~redis.ConnectionPool` implementation does), it
  1425. makes the client wait ("blocks") for a specified number of seconds until
  1426. a connection becomes available.
  1427. Use ``max_connections`` to increase / decrease the pool size::
  1428. >>> pool = BlockingConnectionPool(max_connections=10)
  1429. Use ``timeout`` to tell it either how many seconds to wait for a connection
  1430. to become available, or to block forever:
  1431. >>> # Block forever.
  1432. >>> pool = BlockingConnectionPool(timeout=None)
  1433. >>> # Raise a ``ConnectionError`` after five seconds if a connection is
  1434. >>> # not available.
  1435. >>> pool = BlockingConnectionPool(timeout=5)
  1436. """
  1437. def __init__(
  1438. self,
  1439. max_connections=50,
  1440. timeout=20,
  1441. connection_class=Connection,
  1442. queue_class=LifoQueue,
  1443. **connection_kwargs,
  1444. ):
  1445. self.queue_class = queue_class
  1446. self.timeout = timeout
  1447. super().__init__(
  1448. connection_class=connection_class,
  1449. max_connections=max_connections,
  1450. **connection_kwargs,
  1451. )
  1452. def reset(self):
  1453. # Create and fill up a thread safe queue with ``None`` values.
  1454. self.pool = self.queue_class(self.max_connections)
  1455. while True:
  1456. try:
  1457. self.pool.put_nowait(None)
  1458. except Full:
  1459. break
  1460. # Keep a list of actual connection instances so that we can
  1461. # disconnect them later.
  1462. self._connections = []
  1463. # this must be the last operation in this method. while reset() is
  1464. # called when holding _fork_lock, other threads in this process
  1465. # can call _checkpid() which compares self.pid and os.getpid() without
  1466. # holding any lock (for performance reasons). keeping this assignment
  1467. # as the last operation ensures that those other threads will also
  1468. # notice a pid difference and block waiting for the first thread to
  1469. # release _fork_lock. when each of these threads eventually acquire
  1470. # _fork_lock, they will notice that another thread already called
  1471. # reset() and they will immediately release _fork_lock and continue on.
  1472. self.pid = os.getpid()
  1473. def make_connection(self):
  1474. "Make a fresh connection."
  1475. if self.cache is not None:
  1476. connection = CacheProxyConnection(
  1477. self.connection_class(**self.connection_kwargs), self.cache, self._lock
  1478. )
  1479. else:
  1480. connection = self.connection_class(**self.connection_kwargs)
  1481. self._connections.append(connection)
  1482. return connection
  1483. @deprecated_args(
  1484. args_to_warn=["*"],
  1485. reason="Use get_connection() without args instead",
  1486. version="5.3.0",
  1487. )
  1488. def get_connection(self, command_name=None, *keys, **options):
  1489. """
  1490. Get a connection, blocking for ``self.timeout`` until a connection
  1491. is available from the pool.
  1492. If the connection returned is ``None`` then creates a new connection.
  1493. Because we use a last-in first-out queue, the existing connections
  1494. (having been returned to the pool after the initial ``None`` values
  1495. were added) will be returned before ``None`` values. This means we only
  1496. create new connections when we need to, i.e.: the actual number of
  1497. connections will only increase in response to demand.
  1498. """
  1499. # Make sure we haven't changed process.
  1500. self._checkpid()
  1501. # Try and get a connection from the pool. If one isn't available within
  1502. # self.timeout then raise a ``ConnectionError``.
  1503. connection = None
  1504. try:
  1505. connection = self.pool.get(block=True, timeout=self.timeout)
  1506. except Empty:
  1507. # Note that this is not caught by the redis client and will be
  1508. # raised unless handled by application code. If you want never to
  1509. raise ConnectionError("No connection available.")
  1510. # If the ``connection`` is actually ``None`` then that's a cue to make
  1511. # a new connection to add to the pool.
  1512. if connection is None:
  1513. connection = self.make_connection()
  1514. try:
  1515. # ensure this connection is connected to Redis
  1516. connection.connect()
  1517. # connections that the pool provides should be ready to send
  1518. # a command. if not, the connection was either returned to the
  1519. # pool before all data has been read or the socket has been
  1520. # closed. either way, reconnect and verify everything is good.
  1521. try:
  1522. if connection.can_read():
  1523. raise ConnectionError("Connection has data")
  1524. except (ConnectionError, TimeoutError, OSError):
  1525. connection.disconnect()
  1526. connection.connect()
  1527. if connection.can_read():
  1528. raise ConnectionError("Connection not ready")
  1529. except BaseException:
  1530. # release the connection back to the pool so that we don't leak it
  1531. self.release(connection)
  1532. raise
  1533. return connection
  1534. def release(self, connection):
  1535. "Releases the connection back to the pool."
  1536. # Make sure we haven't changed process.
  1537. self._checkpid()
  1538. if not self.owns_connection(connection):
  1539. # pool doesn't own this connection. do not add it back
  1540. # to the pool. instead add a None value which is a placeholder
  1541. # that will cause the pool to recreate the connection if
  1542. # its needed.
  1543. connection.disconnect()
  1544. self.pool.put_nowait(None)
  1545. return
  1546. # Put the connection back into the pool.
  1547. try:
  1548. self.pool.put_nowait(connection)
  1549. except Full:
  1550. # perhaps the pool has been reset() after a fork? regardless,
  1551. # we don't want this connection
  1552. pass
  1553. def disconnect(self):
  1554. "Disconnects all connections in the pool."
  1555. self._checkpid()
  1556. for connection in self._connections:
  1557. connection.disconnect()