client.py 61 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635
  1. import copy
  2. import re
  3. import threading
  4. import time
  5. from itertools import chain
  6. from typing import (
  7. TYPE_CHECKING,
  8. Any,
  9. Callable,
  10. Dict,
  11. List,
  12. Mapping,
  13. Optional,
  14. Set,
  15. Type,
  16. Union,
  17. )
  18. from redis._parsers.encoders import Encoder
  19. from redis._parsers.helpers import (
  20. _RedisCallbacks,
  21. _RedisCallbacksRESP2,
  22. _RedisCallbacksRESP3,
  23. bool_ok,
  24. )
  25. from redis.backoff import ExponentialWithJitterBackoff
  26. from redis.cache import CacheConfig, CacheInterface
  27. from redis.commands import (
  28. CoreCommands,
  29. RedisModuleCommands,
  30. SentinelCommands,
  31. list_or_args,
  32. )
  33. from redis.commands.core import Script
  34. from redis.connection import (
  35. AbstractConnection,
  36. Connection,
  37. ConnectionPool,
  38. SSLConnection,
  39. UnixDomainSocketConnection,
  40. )
  41. from redis.credentials import CredentialProvider
  42. from redis.event import (
  43. AfterPooledConnectionsInstantiationEvent,
  44. AfterPubSubConnectionInstantiationEvent,
  45. AfterSingleConnectionInstantiationEvent,
  46. ClientType,
  47. EventDispatcher,
  48. )
  49. from redis.exceptions import (
  50. ConnectionError,
  51. ExecAbortError,
  52. PubSubError,
  53. RedisError,
  54. ResponseError,
  55. WatchError,
  56. )
  57. from redis.lock import Lock
  58. from redis.retry import Retry
  59. from redis.utils import (
  60. _set_info_logger,
  61. deprecated_args,
  62. get_lib_version,
  63. safe_str,
  64. str_if_bytes,
  65. truncate_text,
  66. )
  67. if TYPE_CHECKING:
  68. import ssl
  69. import OpenSSL
  70. SYM_EMPTY = b""
  71. EMPTY_RESPONSE = "EMPTY_RESPONSE"
  72. # some responses (ie. dump) are binary, and just meant to never be decoded
  73. NEVER_DECODE = "NEVER_DECODE"
  74. class CaseInsensitiveDict(dict):
  75. "Case insensitive dict implementation. Assumes string keys only."
  76. def __init__(self, data: Dict[str, str]) -> None:
  77. for k, v in data.items():
  78. self[k.upper()] = v
  79. def __contains__(self, k):
  80. return super().__contains__(k.upper())
  81. def __delitem__(self, k):
  82. super().__delitem__(k.upper())
  83. def __getitem__(self, k):
  84. return super().__getitem__(k.upper())
  85. def get(self, k, default=None):
  86. return super().get(k.upper(), default)
  87. def __setitem__(self, k, v):
  88. super().__setitem__(k.upper(), v)
  89. def update(self, data):
  90. data = CaseInsensitiveDict(data)
  91. super().update(data)
  92. class AbstractRedis:
  93. pass
  94. class Redis(RedisModuleCommands, CoreCommands, SentinelCommands):
  95. """
  96. Implementation of the Redis protocol.
  97. This abstract class provides a Python interface to all Redis commands
  98. and an implementation of the Redis protocol.
  99. Pipelines derive from this, implementing how
  100. the commands are sent and received to the Redis server. Based on
  101. configuration, an instance will either use a ConnectionPool, or
  102. Connection object to talk to redis.
  103. It is not safe to pass PubSub or Pipeline objects between threads.
  104. """
  105. @classmethod
  106. def from_url(cls, url: str, **kwargs) -> "Redis":
  107. """
  108. Return a Redis client object configured from the given URL
  109. For example::
  110. redis://[[username]:[password]]@localhost:6379/0
  111. rediss://[[username]:[password]]@localhost:6379/0
  112. unix://[username@]/path/to/socket.sock?db=0[&password=password]
  113. Three URL schemes are supported:
  114. - `redis://` creates a TCP socket connection. See more at:
  115. <https://www.iana.org/assignments/uri-schemes/prov/redis>
  116. - `rediss://` creates a SSL wrapped TCP socket connection. See more at:
  117. <https://www.iana.org/assignments/uri-schemes/prov/rediss>
  118. - ``unix://``: creates a Unix Domain Socket connection.
  119. The username, password, hostname, path and all querystring values
  120. are passed through urllib.parse.unquote in order to replace any
  121. percent-encoded values with their corresponding characters.
  122. There are several ways to specify a database number. The first value
  123. found will be used:
  124. 1. A ``db`` querystring option, e.g. redis://localhost?db=0
  125. 2. If using the redis:// or rediss:// schemes, the path argument
  126. of the url, e.g. redis://localhost/0
  127. 3. A ``db`` keyword argument to this function.
  128. If none of these options are specified, the default db=0 is used.
  129. All querystring options are cast to their appropriate Python types.
  130. Boolean arguments can be specified with string values "True"/"False"
  131. or "Yes"/"No". Values that cannot be properly cast cause a
  132. ``ValueError`` to be raised. Once parsed, the querystring arguments
  133. and keyword arguments are passed to the ``ConnectionPool``'s
  134. class initializer. In the case of conflicting arguments, querystring
  135. arguments always win.
  136. """
  137. single_connection_client = kwargs.pop("single_connection_client", False)
  138. connection_pool = ConnectionPool.from_url(url, **kwargs)
  139. client = cls(
  140. connection_pool=connection_pool,
  141. single_connection_client=single_connection_client,
  142. )
  143. client.auto_close_connection_pool = True
  144. return client
  145. @classmethod
  146. def from_pool(
  147. cls: Type["Redis"],
  148. connection_pool: ConnectionPool,
  149. ) -> "Redis":
  150. """
  151. Return a Redis client from the given connection pool.
  152. The Redis client will take ownership of the connection pool and
  153. close it when the Redis client is closed.
  154. """
  155. client = cls(
  156. connection_pool=connection_pool,
  157. )
  158. client.auto_close_connection_pool = True
  159. return client
  160. @deprecated_args(
  161. args_to_warn=["retry_on_timeout"],
  162. reason="TimeoutError is included by default.",
  163. version="6.0.0",
  164. )
  165. def __init__(
  166. self,
  167. host: str = "localhost",
  168. port: int = 6379,
  169. db: int = 0,
  170. password: Optional[str] = None,
  171. socket_timeout: Optional[float] = None,
  172. socket_connect_timeout: Optional[float] = None,
  173. socket_keepalive: Optional[bool] = None,
  174. socket_keepalive_options: Optional[Mapping[int, Union[int, bytes]]] = None,
  175. connection_pool: Optional[ConnectionPool] = None,
  176. unix_socket_path: Optional[str] = None,
  177. encoding: str = "utf-8",
  178. encoding_errors: str = "strict",
  179. decode_responses: bool = False,
  180. retry_on_timeout: bool = False,
  181. retry: Retry = Retry(
  182. backoff=ExponentialWithJitterBackoff(base=1, cap=10), retries=3
  183. ),
  184. retry_on_error: Optional[List[Type[Exception]]] = None,
  185. ssl: bool = False,
  186. ssl_keyfile: Optional[str] = None,
  187. ssl_certfile: Optional[str] = None,
  188. ssl_cert_reqs: Union[str, "ssl.VerifyMode"] = "required",
  189. ssl_ca_certs: Optional[str] = None,
  190. ssl_ca_path: Optional[str] = None,
  191. ssl_ca_data: Optional[str] = None,
  192. ssl_check_hostname: bool = True,
  193. ssl_password: Optional[str] = None,
  194. ssl_validate_ocsp: bool = False,
  195. ssl_validate_ocsp_stapled: bool = False,
  196. ssl_ocsp_context: Optional["OpenSSL.SSL.Context"] = None,
  197. ssl_ocsp_expected_cert: Optional[str] = None,
  198. ssl_min_version: Optional["ssl.TLSVersion"] = None,
  199. ssl_ciphers: Optional[str] = None,
  200. max_connections: Optional[int] = None,
  201. single_connection_client: bool = False,
  202. health_check_interval: int = 0,
  203. client_name: Optional[str] = None,
  204. lib_name: Optional[str] = "redis-py",
  205. lib_version: Optional[str] = get_lib_version(),
  206. username: Optional[str] = None,
  207. redis_connect_func: Optional[Callable[[], None]] = None,
  208. credential_provider: Optional[CredentialProvider] = None,
  209. protocol: Optional[int] = 2,
  210. cache: Optional[CacheInterface] = None,
  211. cache_config: Optional[CacheConfig] = None,
  212. event_dispatcher: Optional[EventDispatcher] = None,
  213. ) -> None:
  214. """
  215. Initialize a new Redis client.
  216. To specify a retry policy for specific errors, you have two options:
  217. 1. Set the `retry_on_error` to a list of the error/s to retry on, and
  218. you can also set `retry` to a valid `Retry` object(in case the default
  219. one is not appropriate) - with this approach the retries will be triggered
  220. on the default errors specified in the Retry object enriched with the
  221. errors specified in `retry_on_error`.
  222. 2. Define a `Retry` object with configured 'supported_errors' and set
  223. it to the `retry` parameter - with this approach you completely redefine
  224. the errors on which retries will happen.
  225. `retry_on_timeout` is deprecated - please include the TimeoutError
  226. either in the Retry object or in the `retry_on_error` list.
  227. When 'connection_pool' is provided - the retry configuration of the
  228. provided pool will be used.
  229. Args:
  230. single_connection_client:
  231. if `True`, connection pool is not used. In that case `Redis`
  232. instance use is not thread safe.
  233. """
  234. if event_dispatcher is None:
  235. self._event_dispatcher = EventDispatcher()
  236. else:
  237. self._event_dispatcher = event_dispatcher
  238. if not connection_pool:
  239. if not retry_on_error:
  240. retry_on_error = []
  241. kwargs = {
  242. "db": db,
  243. "username": username,
  244. "password": password,
  245. "socket_timeout": socket_timeout,
  246. "encoding": encoding,
  247. "encoding_errors": encoding_errors,
  248. "decode_responses": decode_responses,
  249. "retry_on_error": retry_on_error,
  250. "retry": copy.deepcopy(retry),
  251. "max_connections": max_connections,
  252. "health_check_interval": health_check_interval,
  253. "client_name": client_name,
  254. "lib_name": lib_name,
  255. "lib_version": lib_version,
  256. "redis_connect_func": redis_connect_func,
  257. "credential_provider": credential_provider,
  258. "protocol": protocol,
  259. }
  260. # based on input, setup appropriate connection args
  261. if unix_socket_path is not None:
  262. kwargs.update(
  263. {
  264. "path": unix_socket_path,
  265. "connection_class": UnixDomainSocketConnection,
  266. }
  267. )
  268. else:
  269. # TCP specific options
  270. kwargs.update(
  271. {
  272. "host": host,
  273. "port": port,
  274. "socket_connect_timeout": socket_connect_timeout,
  275. "socket_keepalive": socket_keepalive,
  276. "socket_keepalive_options": socket_keepalive_options,
  277. }
  278. )
  279. if ssl:
  280. kwargs.update(
  281. {
  282. "connection_class": SSLConnection,
  283. "ssl_keyfile": ssl_keyfile,
  284. "ssl_certfile": ssl_certfile,
  285. "ssl_cert_reqs": ssl_cert_reqs,
  286. "ssl_ca_certs": ssl_ca_certs,
  287. "ssl_ca_data": ssl_ca_data,
  288. "ssl_check_hostname": ssl_check_hostname,
  289. "ssl_password": ssl_password,
  290. "ssl_ca_path": ssl_ca_path,
  291. "ssl_validate_ocsp_stapled": ssl_validate_ocsp_stapled,
  292. "ssl_validate_ocsp": ssl_validate_ocsp,
  293. "ssl_ocsp_context": ssl_ocsp_context,
  294. "ssl_ocsp_expected_cert": ssl_ocsp_expected_cert,
  295. "ssl_min_version": ssl_min_version,
  296. "ssl_ciphers": ssl_ciphers,
  297. }
  298. )
  299. if (cache_config or cache) and protocol in [3, "3"]:
  300. kwargs.update(
  301. {
  302. "cache": cache,
  303. "cache_config": cache_config,
  304. }
  305. )
  306. connection_pool = ConnectionPool(**kwargs)
  307. self._event_dispatcher.dispatch(
  308. AfterPooledConnectionsInstantiationEvent(
  309. [connection_pool], ClientType.SYNC, credential_provider
  310. )
  311. )
  312. self.auto_close_connection_pool = True
  313. else:
  314. self.auto_close_connection_pool = False
  315. self._event_dispatcher.dispatch(
  316. AfterPooledConnectionsInstantiationEvent(
  317. [connection_pool], ClientType.SYNC, credential_provider
  318. )
  319. )
  320. self.connection_pool = connection_pool
  321. if (cache_config or cache) and self.connection_pool.get_protocol() not in [
  322. 3,
  323. "3",
  324. ]:
  325. raise RedisError("Client caching is only supported with RESP version 3")
  326. # TODO: To avoid breaking changes during the bug fix, we have to keep non-reentrant lock.
  327. # TODO: Remove this before next major version (7.0.0)
  328. self.single_connection_lock = threading.Lock()
  329. self.connection = None
  330. self._single_connection_client = single_connection_client
  331. if self._single_connection_client:
  332. self.connection = self.connection_pool.get_connection()
  333. self._event_dispatcher.dispatch(
  334. AfterSingleConnectionInstantiationEvent(
  335. self.connection, ClientType.SYNC, self.single_connection_lock
  336. )
  337. )
  338. self.response_callbacks = CaseInsensitiveDict(_RedisCallbacks)
  339. if self.connection_pool.connection_kwargs.get("protocol") in ["3", 3]:
  340. self.response_callbacks.update(_RedisCallbacksRESP3)
  341. else:
  342. self.response_callbacks.update(_RedisCallbacksRESP2)
  343. def __repr__(self) -> str:
  344. return (
  345. f"<{type(self).__module__}.{type(self).__name__}"
  346. f"({repr(self.connection_pool)})>"
  347. )
  348. def get_encoder(self) -> "Encoder":
  349. """Get the connection pool's encoder"""
  350. return self.connection_pool.get_encoder()
  351. def get_connection_kwargs(self) -> Dict:
  352. """Get the connection's key-word arguments"""
  353. return self.connection_pool.connection_kwargs
  354. def get_retry(self) -> Optional[Retry]:
  355. return self.get_connection_kwargs().get("retry")
  356. def set_retry(self, retry: Retry) -> None:
  357. self.get_connection_kwargs().update({"retry": retry})
  358. self.connection_pool.set_retry(retry)
  359. def set_response_callback(self, command: str, callback: Callable) -> None:
  360. """Set a custom Response Callback"""
  361. self.response_callbacks[command] = callback
  362. def load_external_module(self, funcname, func) -> None:
  363. """
  364. This function can be used to add externally defined redis modules,
  365. and their namespaces to the redis client.
  366. funcname - A string containing the name of the function to create
  367. func - The function, being added to this class.
  368. ex: Assume that one has a custom redis module named foomod that
  369. creates command named 'foo.dothing' and 'foo.anotherthing' in redis.
  370. To load function functions into this namespace:
  371. from redis import Redis
  372. from foomodule import F
  373. r = Redis()
  374. r.load_external_module("foo", F)
  375. r.foo().dothing('your', 'arguments')
  376. For a concrete example see the reimport of the redisjson module in
  377. tests/test_connection.py::test_loading_external_modules
  378. """
  379. setattr(self, funcname, func)
  380. def pipeline(self, transaction=True, shard_hint=None) -> "Pipeline":
  381. """
  382. Return a new pipeline object that can queue multiple commands for
  383. later execution. ``transaction`` indicates whether all commands
  384. should be executed atomically. Apart from making a group of operations
  385. atomic, pipelines are useful for reducing the back-and-forth overhead
  386. between the client and server.
  387. """
  388. return Pipeline(
  389. self.connection_pool, self.response_callbacks, transaction, shard_hint
  390. )
  391. def transaction(
  392. self, func: Callable[["Pipeline"], None], *watches, **kwargs
  393. ) -> None:
  394. """
  395. Convenience method for executing the callable `func` as a transaction
  396. while watching all keys specified in `watches`. The 'func' callable
  397. should expect a single argument which is a Pipeline object.
  398. """
  399. shard_hint = kwargs.pop("shard_hint", None)
  400. value_from_callable = kwargs.pop("value_from_callable", False)
  401. watch_delay = kwargs.pop("watch_delay", None)
  402. with self.pipeline(True, shard_hint) as pipe:
  403. while True:
  404. try:
  405. if watches:
  406. pipe.watch(*watches)
  407. func_value = func(pipe)
  408. exec_value = pipe.execute()
  409. return func_value if value_from_callable else exec_value
  410. except WatchError:
  411. if watch_delay is not None and watch_delay > 0:
  412. time.sleep(watch_delay)
  413. continue
  414. def lock(
  415. self,
  416. name: str,
  417. timeout: Optional[float] = None,
  418. sleep: float = 0.1,
  419. blocking: bool = True,
  420. blocking_timeout: Optional[float] = None,
  421. lock_class: Union[None, Any] = None,
  422. thread_local: bool = True,
  423. raise_on_release_error: bool = True,
  424. ):
  425. """
  426. Return a new Lock object using key ``name`` that mimics
  427. the behavior of threading.Lock.
  428. If specified, ``timeout`` indicates a maximum life for the lock.
  429. By default, it will remain locked until release() is called.
  430. ``sleep`` indicates the amount of time to sleep per loop iteration
  431. when the lock is in blocking mode and another client is currently
  432. holding the lock.
  433. ``blocking`` indicates whether calling ``acquire`` should block until
  434. the lock has been acquired or to fail immediately, causing ``acquire``
  435. to return False and the lock not being acquired. Defaults to True.
  436. Note this value can be overridden by passing a ``blocking``
  437. argument to ``acquire``.
  438. ``blocking_timeout`` indicates the maximum amount of time in seconds to
  439. spend trying to acquire the lock. A value of ``None`` indicates
  440. continue trying forever. ``blocking_timeout`` can be specified as a
  441. float or integer, both representing the number of seconds to wait.
  442. ``lock_class`` forces the specified lock implementation. Note that as
  443. of redis-py 3.0, the only lock class we implement is ``Lock`` (which is
  444. a Lua-based lock). So, it's unlikely you'll need this parameter, unless
  445. you have created your own custom lock class.
  446. ``thread_local`` indicates whether the lock token is placed in
  447. thread-local storage. By default, the token is placed in thread local
  448. storage so that a thread only sees its token, not a token set by
  449. another thread. Consider the following timeline:
  450. time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds.
  451. thread-1 sets the token to "abc"
  452. time: 1, thread-2 blocks trying to acquire `my-lock` using the
  453. Lock instance.
  454. time: 5, thread-1 has not yet completed. redis expires the lock
  455. key.
  456. time: 5, thread-2 acquired `my-lock` now that it's available.
  457. thread-2 sets the token to "xyz"
  458. time: 6, thread-1 finishes its work and calls release(). if the
  459. token is *not* stored in thread local storage, then
  460. thread-1 would see the token value as "xyz" and would be
  461. able to successfully release the thread-2's lock.
  462. ``raise_on_release_error`` indicates whether to raise an exception when
  463. the lock is no longer owned when exiting the context manager. By default,
  464. this is True, meaning an exception will be raised. If False, the warning
  465. will be logged and the exception will be suppressed.
  466. In some use cases it's necessary to disable thread local storage. For
  467. example, if you have code where one thread acquires a lock and passes
  468. that lock instance to a worker thread to release later. If thread
  469. local storage isn't disabled in this case, the worker thread won't see
  470. the token set by the thread that acquired the lock. Our assumption
  471. is that these cases aren't common and as such default to using
  472. thread local storage."""
  473. if lock_class is None:
  474. lock_class = Lock
  475. return lock_class(
  476. self,
  477. name,
  478. timeout=timeout,
  479. sleep=sleep,
  480. blocking=blocking,
  481. blocking_timeout=blocking_timeout,
  482. thread_local=thread_local,
  483. raise_on_release_error=raise_on_release_error,
  484. )
  485. def pubsub(self, **kwargs):
  486. """
  487. Return a Publish/Subscribe object. With this object, you can
  488. subscribe to channels and listen for messages that get published to
  489. them.
  490. """
  491. return PubSub(
  492. self.connection_pool, event_dispatcher=self._event_dispatcher, **kwargs
  493. )
  494. def monitor(self):
  495. return Monitor(self.connection_pool)
  496. def client(self):
  497. return self.__class__(
  498. connection_pool=self.connection_pool, single_connection_client=True
  499. )
  500. def __enter__(self):
  501. return self
  502. def __exit__(self, exc_type, exc_value, traceback):
  503. self.close()
  504. def __del__(self):
  505. try:
  506. self.close()
  507. except Exception:
  508. pass
  509. def close(self) -> None:
  510. # In case a connection property does not yet exist
  511. # (due to a crash earlier in the Redis() constructor), return
  512. # immediately as there is nothing to clean-up.
  513. if not hasattr(self, "connection"):
  514. return
  515. conn = self.connection
  516. if conn:
  517. self.connection = None
  518. self.connection_pool.release(conn)
  519. if self.auto_close_connection_pool:
  520. self.connection_pool.disconnect()
  521. def _send_command_parse_response(self, conn, command_name, *args, **options):
  522. """
  523. Send a command and parse the response
  524. """
  525. conn.send_command(*args, **options)
  526. return self.parse_response(conn, command_name, **options)
  527. def _close_connection(self, conn) -> None:
  528. """
  529. Close the connection before retrying.
  530. The supported exceptions are already checked in the
  531. retry object so we don't need to do it here.
  532. After we disconnect the connection, it will try to reconnect and
  533. do a health check as part of the send_command logic(on connection level).
  534. """
  535. conn.disconnect()
  536. # COMMAND EXECUTION AND PROTOCOL PARSING
  537. def execute_command(self, *args, **options):
  538. return self._execute_command(*args, **options)
  539. def _execute_command(self, *args, **options):
  540. """Execute a command and return a parsed response"""
  541. pool = self.connection_pool
  542. command_name = args[0]
  543. conn = self.connection or pool.get_connection()
  544. if self._single_connection_client:
  545. self.single_connection_lock.acquire()
  546. try:
  547. return conn.retry.call_with_retry(
  548. lambda: self._send_command_parse_response(
  549. conn, command_name, *args, **options
  550. ),
  551. lambda _: self._close_connection(conn),
  552. )
  553. finally:
  554. if self._single_connection_client:
  555. self.single_connection_lock.release()
  556. if not self.connection:
  557. pool.release(conn)
  558. def parse_response(self, connection, command_name, **options):
  559. """Parses a response from the Redis server"""
  560. try:
  561. if NEVER_DECODE in options:
  562. response = connection.read_response(disable_decoding=True)
  563. options.pop(NEVER_DECODE)
  564. else:
  565. response = connection.read_response()
  566. except ResponseError:
  567. if EMPTY_RESPONSE in options:
  568. return options[EMPTY_RESPONSE]
  569. raise
  570. if EMPTY_RESPONSE in options:
  571. options.pop(EMPTY_RESPONSE)
  572. # Remove keys entry, it needs only for cache.
  573. options.pop("keys", None)
  574. if command_name in self.response_callbacks:
  575. return self.response_callbacks[command_name](response, **options)
  576. return response
  577. def get_cache(self) -> Optional[CacheInterface]:
  578. return self.connection_pool.cache
  579. StrictRedis = Redis
  580. class Monitor:
  581. """
  582. Monitor is useful for handling the MONITOR command to the redis server.
  583. next_command() method returns one command from monitor
  584. listen() method yields commands from monitor.
  585. """
  586. monitor_re = re.compile(r"\[(\d+) (.*?)\] (.*)")
  587. command_re = re.compile(r'"(.*?)(?<!\\)"')
  588. def __init__(self, connection_pool):
  589. self.connection_pool = connection_pool
  590. self.connection = self.connection_pool.get_connection()
  591. def __enter__(self):
  592. self.connection.send_command("MONITOR")
  593. # check that monitor returns 'OK', but don't return it to user
  594. response = self.connection.read_response()
  595. if not bool_ok(response):
  596. raise RedisError(f"MONITOR failed: {response}")
  597. return self
  598. def __exit__(self, *args):
  599. self.connection.disconnect()
  600. self.connection_pool.release(self.connection)
  601. def next_command(self):
  602. """Parse the response from a monitor command"""
  603. response = self.connection.read_response()
  604. if isinstance(response, bytes):
  605. response = self.connection.encoder.decode(response, force=True)
  606. command_time, command_data = response.split(" ", 1)
  607. m = self.monitor_re.match(command_data)
  608. db_id, client_info, command = m.groups()
  609. command = " ".join(self.command_re.findall(command))
  610. # Redis escapes double quotes because each piece of the command
  611. # string is surrounded by double quotes. We don't have that
  612. # requirement so remove the escaping and leave the quote.
  613. command = command.replace('\\"', '"')
  614. if client_info == "lua":
  615. client_address = "lua"
  616. client_port = ""
  617. client_type = "lua"
  618. elif client_info.startswith("unix"):
  619. client_address = "unix"
  620. client_port = client_info[5:]
  621. client_type = "unix"
  622. else:
  623. # use rsplit as ipv6 addresses contain colons
  624. client_address, client_port = client_info.rsplit(":", 1)
  625. client_type = "tcp"
  626. return {
  627. "time": float(command_time),
  628. "db": int(db_id),
  629. "client_address": client_address,
  630. "client_port": client_port,
  631. "client_type": client_type,
  632. "command": command,
  633. }
  634. def listen(self):
  635. """Listen for commands coming to the server."""
  636. while True:
  637. yield self.next_command()
  638. class PubSub:
  639. """
  640. PubSub provides publish, subscribe and listen support to Redis channels.
  641. After subscribing to one or more channels, the listen() method will block
  642. until a message arrives on one of the subscribed channels. That message
  643. will be returned and it's safe to start listening again.
  644. """
  645. PUBLISH_MESSAGE_TYPES = ("message", "pmessage", "smessage")
  646. UNSUBSCRIBE_MESSAGE_TYPES = ("unsubscribe", "punsubscribe", "sunsubscribe")
  647. HEALTH_CHECK_MESSAGE = "redis-py-health-check"
  648. def __init__(
  649. self,
  650. connection_pool,
  651. shard_hint=None,
  652. ignore_subscribe_messages: bool = False,
  653. encoder: Optional["Encoder"] = None,
  654. push_handler_func: Union[None, Callable[[str], None]] = None,
  655. event_dispatcher: Optional["EventDispatcher"] = None,
  656. ):
  657. self.connection_pool = connection_pool
  658. self.shard_hint = shard_hint
  659. self.ignore_subscribe_messages = ignore_subscribe_messages
  660. self.connection = None
  661. self.subscribed_event = threading.Event()
  662. # we need to know the encoding options for this connection in order
  663. # to lookup channel and pattern names for callback handlers.
  664. self.encoder = encoder
  665. self.push_handler_func = push_handler_func
  666. if event_dispatcher is None:
  667. self._event_dispatcher = EventDispatcher()
  668. else:
  669. self._event_dispatcher = event_dispatcher
  670. # TODO: To avoid breaking changes during the bug fix, we have to keep non-reentrant lock.
  671. # TODO: Remove this before next major version (7.0.0)
  672. self._lock = threading.Lock()
  673. if self.encoder is None:
  674. self.encoder = self.connection_pool.get_encoder()
  675. self.health_check_response_b = self.encoder.encode(self.HEALTH_CHECK_MESSAGE)
  676. if self.encoder.decode_responses:
  677. self.health_check_response = ["pong", self.HEALTH_CHECK_MESSAGE]
  678. else:
  679. self.health_check_response = [b"pong", self.health_check_response_b]
  680. if self.push_handler_func is None:
  681. _set_info_logger()
  682. self.reset()
  683. def __enter__(self) -> "PubSub":
  684. return self
  685. def __exit__(self, exc_type, exc_value, traceback) -> None:
  686. self.reset()
  687. def __del__(self) -> None:
  688. try:
  689. # if this object went out of scope prior to shutting down
  690. # subscriptions, close the connection manually before
  691. # returning it to the connection pool
  692. self.reset()
  693. except Exception:
  694. pass
  695. def reset(self) -> None:
  696. if self.connection:
  697. self.connection.disconnect()
  698. self.connection.deregister_connect_callback(self.on_connect)
  699. self.connection_pool.release(self.connection)
  700. self.connection = None
  701. self.health_check_response_counter = 0
  702. self.channels = {}
  703. self.pending_unsubscribe_channels = set()
  704. self.shard_channels = {}
  705. self.pending_unsubscribe_shard_channels = set()
  706. self.patterns = {}
  707. self.pending_unsubscribe_patterns = set()
  708. self.subscribed_event.clear()
  709. def close(self) -> None:
  710. self.reset()
  711. def on_connect(self, connection) -> None:
  712. "Re-subscribe to any channels and patterns previously subscribed to"
  713. # NOTE: for python3, we can't pass bytestrings as keyword arguments
  714. # so we need to decode channel/pattern names back to unicode strings
  715. # before passing them to [p]subscribe.
  716. self.pending_unsubscribe_channels.clear()
  717. self.pending_unsubscribe_patterns.clear()
  718. self.pending_unsubscribe_shard_channels.clear()
  719. if self.channels:
  720. channels = {
  721. self.encoder.decode(k, force=True): v for k, v in self.channels.items()
  722. }
  723. self.subscribe(**channels)
  724. if self.patterns:
  725. patterns = {
  726. self.encoder.decode(k, force=True): v for k, v in self.patterns.items()
  727. }
  728. self.psubscribe(**patterns)
  729. if self.shard_channels:
  730. shard_channels = {
  731. self.encoder.decode(k, force=True): v
  732. for k, v in self.shard_channels.items()
  733. }
  734. self.ssubscribe(**shard_channels)
  735. @property
  736. def subscribed(self) -> bool:
  737. """Indicates if there are subscriptions to any channels or patterns"""
  738. return self.subscribed_event.is_set()
  739. def execute_command(self, *args):
  740. """Execute a publish/subscribe command"""
  741. # NOTE: don't parse the response in this function -- it could pull a
  742. # legitimate message off the stack if the connection is already
  743. # subscribed to one or more channels
  744. if self.connection is None:
  745. self.connection = self.connection_pool.get_connection()
  746. # register a callback that re-subscribes to any channels we
  747. # were listening to when we were disconnected
  748. self.connection.register_connect_callback(self.on_connect)
  749. if self.push_handler_func is not None:
  750. self.connection._parser.set_pubsub_push_handler(self.push_handler_func)
  751. self._event_dispatcher.dispatch(
  752. AfterPubSubConnectionInstantiationEvent(
  753. self.connection, self.connection_pool, ClientType.SYNC, self._lock
  754. )
  755. )
  756. connection = self.connection
  757. kwargs = {"check_health": not self.subscribed}
  758. if not self.subscribed:
  759. self.clean_health_check_responses()
  760. with self._lock:
  761. self._execute(connection, connection.send_command, *args, **kwargs)
  762. def clean_health_check_responses(self) -> None:
  763. """
  764. If any health check responses are present, clean them
  765. """
  766. ttl = 10
  767. conn = self.connection
  768. while self.health_check_response_counter > 0 and ttl > 0:
  769. if self._execute(conn, conn.can_read, timeout=conn.socket_timeout):
  770. response = self._execute(conn, conn.read_response)
  771. if self.is_health_check_response(response):
  772. self.health_check_response_counter -= 1
  773. else:
  774. raise PubSubError(
  775. "A non health check response was cleaned by "
  776. "execute_command: {}".format(response)
  777. )
  778. ttl -= 1
  779. def _reconnect(self, conn) -> None:
  780. """
  781. The supported exceptions are already checked in the
  782. retry object so we don't need to do it here.
  783. In this error handler we are trying to reconnect to the server.
  784. """
  785. conn.disconnect()
  786. conn.connect()
  787. def _execute(self, conn, command, *args, **kwargs):
  788. """
  789. Connect manually upon disconnection. If the Redis server is down,
  790. this will fail and raise a ConnectionError as desired.
  791. After reconnection, the ``on_connect`` callback should have been
  792. called by the # connection to resubscribe us to any channels and
  793. patterns we were previously listening to
  794. """
  795. return conn.retry.call_with_retry(
  796. lambda: command(*args, **kwargs),
  797. lambda _: self._reconnect(conn),
  798. )
  799. def parse_response(self, block=True, timeout=0):
  800. """Parse the response from a publish/subscribe command"""
  801. conn = self.connection
  802. if conn is None:
  803. raise RuntimeError(
  804. "pubsub connection not set: "
  805. "did you forget to call subscribe() or psubscribe()?"
  806. )
  807. self.check_health()
  808. def try_read():
  809. if not block:
  810. if not conn.can_read(timeout=timeout):
  811. return None
  812. else:
  813. conn.connect()
  814. return conn.read_response(disconnect_on_error=False, push_request=True)
  815. response = self._execute(conn, try_read)
  816. if self.is_health_check_response(response):
  817. # ignore the health check message as user might not expect it
  818. self.health_check_response_counter -= 1
  819. return None
  820. return response
  821. def is_health_check_response(self, response) -> bool:
  822. """
  823. Check if the response is a health check response.
  824. If there are no subscriptions redis responds to PING command with a
  825. bulk response, instead of a multi-bulk with "pong" and the response.
  826. """
  827. return response in [
  828. self.health_check_response, # If there was a subscription
  829. self.health_check_response_b, # If there wasn't
  830. ]
  831. def check_health(self) -> None:
  832. conn = self.connection
  833. if conn is None:
  834. raise RuntimeError(
  835. "pubsub connection not set: "
  836. "did you forget to call subscribe() or psubscribe()?"
  837. )
  838. if conn.health_check_interval and time.monotonic() > conn.next_health_check:
  839. conn.send_command("PING", self.HEALTH_CHECK_MESSAGE, check_health=False)
  840. self.health_check_response_counter += 1
  841. def _normalize_keys(self, data) -> Dict:
  842. """
  843. normalize channel/pattern names to be either bytes or strings
  844. based on whether responses are automatically decoded. this saves us
  845. from coercing the value for each message coming in.
  846. """
  847. encode = self.encoder.encode
  848. decode = self.encoder.decode
  849. return {decode(encode(k)): v for k, v in data.items()}
  850. def psubscribe(self, *args, **kwargs):
  851. """
  852. Subscribe to channel patterns. Patterns supplied as keyword arguments
  853. expect a pattern name as the key and a callable as the value. A
  854. pattern's callable will be invoked automatically when a message is
  855. received on that pattern rather than producing a message via
  856. ``listen()``.
  857. """
  858. if args:
  859. args = list_or_args(args[0], args[1:])
  860. new_patterns = dict.fromkeys(args)
  861. new_patterns.update(kwargs)
  862. ret_val = self.execute_command("PSUBSCRIBE", *new_patterns.keys())
  863. # update the patterns dict AFTER we send the command. we don't want to
  864. # subscribe twice to these patterns, once for the command and again
  865. # for the reconnection.
  866. new_patterns = self._normalize_keys(new_patterns)
  867. self.patterns.update(new_patterns)
  868. if not self.subscribed:
  869. # Set the subscribed_event flag to True
  870. self.subscribed_event.set()
  871. # Clear the health check counter
  872. self.health_check_response_counter = 0
  873. self.pending_unsubscribe_patterns.difference_update(new_patterns)
  874. return ret_val
  875. def punsubscribe(self, *args):
  876. """
  877. Unsubscribe from the supplied patterns. If empty, unsubscribe from
  878. all patterns.
  879. """
  880. if args:
  881. args = list_or_args(args[0], args[1:])
  882. patterns = self._normalize_keys(dict.fromkeys(args))
  883. else:
  884. patterns = self.patterns
  885. self.pending_unsubscribe_patterns.update(patterns)
  886. return self.execute_command("PUNSUBSCRIBE", *args)
  887. def subscribe(self, *args, **kwargs):
  888. """
  889. Subscribe to channels. Channels supplied as keyword arguments expect
  890. a channel name as the key and a callable as the value. A channel's
  891. callable will be invoked automatically when a message is received on
  892. that channel rather than producing a message via ``listen()`` or
  893. ``get_message()``.
  894. """
  895. if args:
  896. args = list_or_args(args[0], args[1:])
  897. new_channels = dict.fromkeys(args)
  898. new_channels.update(kwargs)
  899. ret_val = self.execute_command("SUBSCRIBE", *new_channels.keys())
  900. # update the channels dict AFTER we send the command. we don't want to
  901. # subscribe twice to these channels, once for the command and again
  902. # for the reconnection.
  903. new_channels = self._normalize_keys(new_channels)
  904. self.channels.update(new_channels)
  905. if not self.subscribed:
  906. # Set the subscribed_event flag to True
  907. self.subscribed_event.set()
  908. # Clear the health check counter
  909. self.health_check_response_counter = 0
  910. self.pending_unsubscribe_channels.difference_update(new_channels)
  911. return ret_val
  912. def unsubscribe(self, *args):
  913. """
  914. Unsubscribe from the supplied channels. If empty, unsubscribe from
  915. all channels
  916. """
  917. if args:
  918. args = list_or_args(args[0], args[1:])
  919. channels = self._normalize_keys(dict.fromkeys(args))
  920. else:
  921. channels = self.channels
  922. self.pending_unsubscribe_channels.update(channels)
  923. return self.execute_command("UNSUBSCRIBE", *args)
  924. def ssubscribe(self, *args, target_node=None, **kwargs):
  925. """
  926. Subscribes the client to the specified shard channels.
  927. Channels supplied as keyword arguments expect a channel name as the key
  928. and a callable as the value. A channel's callable will be invoked automatically
  929. when a message is received on that channel rather than producing a message via
  930. ``listen()`` or ``get_sharded_message()``.
  931. """
  932. if args:
  933. args = list_or_args(args[0], args[1:])
  934. new_s_channels = dict.fromkeys(args)
  935. new_s_channels.update(kwargs)
  936. ret_val = self.execute_command("SSUBSCRIBE", *new_s_channels.keys())
  937. # update the s_channels dict AFTER we send the command. we don't want to
  938. # subscribe twice to these channels, once for the command and again
  939. # for the reconnection.
  940. new_s_channels = self._normalize_keys(new_s_channels)
  941. self.shard_channels.update(new_s_channels)
  942. if not self.subscribed:
  943. # Set the subscribed_event flag to True
  944. self.subscribed_event.set()
  945. # Clear the health check counter
  946. self.health_check_response_counter = 0
  947. self.pending_unsubscribe_shard_channels.difference_update(new_s_channels)
  948. return ret_val
  949. def sunsubscribe(self, *args, target_node=None):
  950. """
  951. Unsubscribe from the supplied shard_channels. If empty, unsubscribe from
  952. all shard_channels
  953. """
  954. if args:
  955. args = list_or_args(args[0], args[1:])
  956. s_channels = self._normalize_keys(dict.fromkeys(args))
  957. else:
  958. s_channels = self.shard_channels
  959. self.pending_unsubscribe_shard_channels.update(s_channels)
  960. return self.execute_command("SUNSUBSCRIBE", *args)
  961. def listen(self):
  962. "Listen for messages on channels this client has been subscribed to"
  963. while self.subscribed:
  964. response = self.handle_message(self.parse_response(block=True))
  965. if response is not None:
  966. yield response
  967. def get_message(
  968. self, ignore_subscribe_messages: bool = False, timeout: float = 0.0
  969. ):
  970. """
  971. Get the next message if one is available, otherwise None.
  972. If timeout is specified, the system will wait for `timeout` seconds
  973. before returning. Timeout should be specified as a floating point
  974. number, or None, to wait indefinitely.
  975. """
  976. if not self.subscribed:
  977. # Wait for subscription
  978. start_time = time.monotonic()
  979. if self.subscribed_event.wait(timeout) is True:
  980. # The connection was subscribed during the timeout time frame.
  981. # The timeout should be adjusted based on the time spent
  982. # waiting for the subscription
  983. time_spent = time.monotonic() - start_time
  984. timeout = max(0.0, timeout - time_spent)
  985. else:
  986. # The connection isn't subscribed to any channels or patterns,
  987. # so no messages are available
  988. return None
  989. response = self.parse_response(block=(timeout is None), timeout=timeout)
  990. if response:
  991. return self.handle_message(response, ignore_subscribe_messages)
  992. return None
  993. get_sharded_message = get_message
  994. def ping(self, message: Union[str, None] = None) -> bool:
  995. """
  996. Ping the Redis server
  997. """
  998. args = ["PING", message] if message is not None else ["PING"]
  999. return self.execute_command(*args)
  1000. def handle_message(self, response, ignore_subscribe_messages=False):
  1001. """
  1002. Parses a pub/sub message. If the channel or pattern was subscribed to
  1003. with a message handler, the handler is invoked instead of a parsed
  1004. message being returned.
  1005. """
  1006. if response is None:
  1007. return None
  1008. if isinstance(response, bytes):
  1009. response = [b"pong", response] if response != b"PONG" else [b"pong", b""]
  1010. message_type = str_if_bytes(response[0])
  1011. if message_type == "pmessage":
  1012. message = {
  1013. "type": message_type,
  1014. "pattern": response[1],
  1015. "channel": response[2],
  1016. "data": response[3],
  1017. }
  1018. elif message_type == "pong":
  1019. message = {
  1020. "type": message_type,
  1021. "pattern": None,
  1022. "channel": None,
  1023. "data": response[1],
  1024. }
  1025. else:
  1026. message = {
  1027. "type": message_type,
  1028. "pattern": None,
  1029. "channel": response[1],
  1030. "data": response[2],
  1031. }
  1032. # if this is an unsubscribe message, remove it from memory
  1033. if message_type in self.UNSUBSCRIBE_MESSAGE_TYPES:
  1034. if message_type == "punsubscribe":
  1035. pattern = response[1]
  1036. if pattern in self.pending_unsubscribe_patterns:
  1037. self.pending_unsubscribe_patterns.remove(pattern)
  1038. self.patterns.pop(pattern, None)
  1039. elif message_type == "sunsubscribe":
  1040. s_channel = response[1]
  1041. if s_channel in self.pending_unsubscribe_shard_channels:
  1042. self.pending_unsubscribe_shard_channels.remove(s_channel)
  1043. self.shard_channels.pop(s_channel, None)
  1044. else:
  1045. channel = response[1]
  1046. if channel in self.pending_unsubscribe_channels:
  1047. self.pending_unsubscribe_channels.remove(channel)
  1048. self.channels.pop(channel, None)
  1049. if not self.channels and not self.patterns and not self.shard_channels:
  1050. # There are no subscriptions anymore, set subscribed_event flag
  1051. # to false
  1052. self.subscribed_event.clear()
  1053. if message_type in self.PUBLISH_MESSAGE_TYPES:
  1054. # if there's a message handler, invoke it
  1055. if message_type == "pmessage":
  1056. handler = self.patterns.get(message["pattern"], None)
  1057. elif message_type == "smessage":
  1058. handler = self.shard_channels.get(message["channel"], None)
  1059. else:
  1060. handler = self.channels.get(message["channel"], None)
  1061. if handler:
  1062. handler(message)
  1063. return None
  1064. elif message_type != "pong":
  1065. # this is a subscribe/unsubscribe message. ignore if we don't
  1066. # want them
  1067. if ignore_subscribe_messages or self.ignore_subscribe_messages:
  1068. return None
  1069. return message
  1070. def run_in_thread(
  1071. self,
  1072. sleep_time: float = 0.0,
  1073. daemon: bool = False,
  1074. exception_handler: Optional[Callable] = None,
  1075. ) -> "PubSubWorkerThread":
  1076. for channel, handler in self.channels.items():
  1077. if handler is None:
  1078. raise PubSubError(f"Channel: '{channel}' has no handler registered")
  1079. for pattern, handler in self.patterns.items():
  1080. if handler is None:
  1081. raise PubSubError(f"Pattern: '{pattern}' has no handler registered")
  1082. for s_channel, handler in self.shard_channels.items():
  1083. if handler is None:
  1084. raise PubSubError(
  1085. f"Shard Channel: '{s_channel}' has no handler registered"
  1086. )
  1087. thread = PubSubWorkerThread(
  1088. self, sleep_time, daemon=daemon, exception_handler=exception_handler
  1089. )
  1090. thread.start()
  1091. return thread
  1092. class PubSubWorkerThread(threading.Thread):
  1093. def __init__(
  1094. self,
  1095. pubsub,
  1096. sleep_time: float,
  1097. daemon: bool = False,
  1098. exception_handler: Union[
  1099. Callable[[Exception, "PubSub", "PubSubWorkerThread"], None], None
  1100. ] = None,
  1101. ):
  1102. super().__init__()
  1103. self.daemon = daemon
  1104. self.pubsub = pubsub
  1105. self.sleep_time = sleep_time
  1106. self.exception_handler = exception_handler
  1107. self._running = threading.Event()
  1108. def run(self) -> None:
  1109. if self._running.is_set():
  1110. return
  1111. self._running.set()
  1112. pubsub = self.pubsub
  1113. sleep_time = self.sleep_time
  1114. while self._running.is_set():
  1115. try:
  1116. pubsub.get_message(ignore_subscribe_messages=True, timeout=sleep_time)
  1117. except BaseException as e:
  1118. if self.exception_handler is None:
  1119. raise
  1120. self.exception_handler(e, pubsub, self)
  1121. pubsub.close()
  1122. def stop(self) -> None:
  1123. # trip the flag so the run loop exits. the run loop will
  1124. # close the pubsub connection, which disconnects the socket
  1125. # and returns the connection to the pool.
  1126. self._running.clear()
  1127. class Pipeline(Redis):
  1128. """
  1129. Pipelines provide a way to transmit multiple commands to the Redis server
  1130. in one transmission. This is convenient for batch processing, such as
  1131. saving all the values in a list to Redis.
  1132. All commands executed within a pipeline(when running in transactional mode,
  1133. which is the default behavior) are wrapped with MULTI and EXEC
  1134. calls. This guarantees all commands executed in the pipeline will be
  1135. executed atomically.
  1136. Any command raising an exception does *not* halt the execution of
  1137. subsequent commands in the pipeline. Instead, the exception is caught
  1138. and its instance is placed into the response list returned by execute().
  1139. Code iterating over the response list should be able to deal with an
  1140. instance of an exception as a potential value. In general, these will be
  1141. ResponseError exceptions, such as those raised when issuing a command
  1142. on a key of a different datatype.
  1143. """
  1144. UNWATCH_COMMANDS = {"DISCARD", "EXEC", "UNWATCH"}
  1145. def __init__(
  1146. self,
  1147. connection_pool: ConnectionPool,
  1148. response_callbacks,
  1149. transaction,
  1150. shard_hint,
  1151. ):
  1152. self.connection_pool = connection_pool
  1153. self.connection: Optional[Connection] = None
  1154. self.response_callbacks = response_callbacks
  1155. self.transaction = transaction
  1156. self.shard_hint = shard_hint
  1157. self.watching = False
  1158. self.command_stack = []
  1159. self.scripts: Set[Script] = set()
  1160. self.explicit_transaction = False
  1161. def __enter__(self) -> "Pipeline":
  1162. return self
  1163. def __exit__(self, exc_type, exc_value, traceback):
  1164. self.reset()
  1165. def __del__(self):
  1166. try:
  1167. self.reset()
  1168. except Exception:
  1169. pass
  1170. def __len__(self) -> int:
  1171. return len(self.command_stack)
  1172. def __bool__(self) -> bool:
  1173. """Pipeline instances should always evaluate to True"""
  1174. return True
  1175. def reset(self) -> None:
  1176. self.command_stack = []
  1177. self.scripts = set()
  1178. # make sure to reset the connection state in the event that we were
  1179. # watching something
  1180. if self.watching and self.connection:
  1181. try:
  1182. # call this manually since our unwatch or
  1183. # immediate_execute_command methods can call reset()
  1184. self.connection.send_command("UNWATCH")
  1185. self.connection.read_response()
  1186. except ConnectionError:
  1187. # disconnect will also remove any previous WATCHes
  1188. self.connection.disconnect()
  1189. # clean up the other instance attributes
  1190. self.watching = False
  1191. self.explicit_transaction = False
  1192. # we can safely return the connection to the pool here since we're
  1193. # sure we're no longer WATCHing anything
  1194. if self.connection:
  1195. self.connection_pool.release(self.connection)
  1196. self.connection = None
  1197. def close(self) -> None:
  1198. """Close the pipeline"""
  1199. self.reset()
  1200. def multi(self) -> None:
  1201. """
  1202. Start a transactional block of the pipeline after WATCH commands
  1203. are issued. End the transactional block with `execute`.
  1204. """
  1205. if self.explicit_transaction:
  1206. raise RedisError("Cannot issue nested calls to MULTI")
  1207. if self.command_stack:
  1208. raise RedisError(
  1209. "Commands without an initial WATCH have already been issued"
  1210. )
  1211. self.explicit_transaction = True
  1212. def execute_command(self, *args, **kwargs):
  1213. if (self.watching or args[0] == "WATCH") and not self.explicit_transaction:
  1214. return self.immediate_execute_command(*args, **kwargs)
  1215. return self.pipeline_execute_command(*args, **kwargs)
  1216. def _disconnect_reset_raise_on_watching(
  1217. self,
  1218. conn: AbstractConnection,
  1219. error: Exception,
  1220. ) -> None:
  1221. """
  1222. Close the connection reset watching state and
  1223. raise an exception if we were watching.
  1224. The supported exceptions are already checked in the
  1225. retry object so we don't need to do it here.
  1226. After we disconnect the connection, it will try to reconnect and
  1227. do a health check as part of the send_command logic(on connection level).
  1228. """
  1229. conn.disconnect()
  1230. # if we were already watching a variable, the watch is no longer
  1231. # valid since this connection has died. raise a WatchError, which
  1232. # indicates the user should retry this transaction.
  1233. if self.watching:
  1234. self.reset()
  1235. raise WatchError(
  1236. f"A {type(error).__name__} occurred while watching one or more keys"
  1237. )
  1238. def immediate_execute_command(self, *args, **options):
  1239. """
  1240. Execute a command immediately, but don't auto-retry on the supported
  1241. errors for retry if we're already WATCHing a variable.
  1242. Used when issuing WATCH or subsequent commands retrieving their values but before
  1243. MULTI is called.
  1244. """
  1245. command_name = args[0]
  1246. conn = self.connection
  1247. # if this is the first call, we need a connection
  1248. if not conn:
  1249. conn = self.connection_pool.get_connection()
  1250. self.connection = conn
  1251. return conn.retry.call_with_retry(
  1252. lambda: self._send_command_parse_response(
  1253. conn, command_name, *args, **options
  1254. ),
  1255. lambda error: self._disconnect_reset_raise_on_watching(conn, error),
  1256. )
  1257. def pipeline_execute_command(self, *args, **options) -> "Pipeline":
  1258. """
  1259. Stage a command to be executed when execute() is next called
  1260. Returns the current Pipeline object back so commands can be
  1261. chained together, such as:
  1262. pipe = pipe.set('foo', 'bar').incr('baz').decr('bang')
  1263. At some other point, you can then run: pipe.execute(),
  1264. which will execute all commands queued in the pipe.
  1265. """
  1266. self.command_stack.append((args, options))
  1267. return self
  1268. def _execute_transaction(
  1269. self, connection: Connection, commands, raise_on_error
  1270. ) -> List:
  1271. cmds = chain([(("MULTI",), {})], commands, [(("EXEC",), {})])
  1272. all_cmds = connection.pack_commands(
  1273. [args for args, options in cmds if EMPTY_RESPONSE not in options]
  1274. )
  1275. connection.send_packed_command(all_cmds)
  1276. errors = []
  1277. # parse off the response for MULTI
  1278. # NOTE: we need to handle ResponseErrors here and continue
  1279. # so that we read all the additional command messages from
  1280. # the socket
  1281. try:
  1282. self.parse_response(connection, "_")
  1283. except ResponseError as e:
  1284. errors.append((0, e))
  1285. # and all the other commands
  1286. for i, command in enumerate(commands):
  1287. if EMPTY_RESPONSE in command[1]:
  1288. errors.append((i, command[1][EMPTY_RESPONSE]))
  1289. else:
  1290. try:
  1291. self.parse_response(connection, "_")
  1292. except ResponseError as e:
  1293. self.annotate_exception(e, i + 1, command[0])
  1294. errors.append((i, e))
  1295. # parse the EXEC.
  1296. try:
  1297. response = self.parse_response(connection, "_")
  1298. except ExecAbortError:
  1299. if errors:
  1300. raise errors[0][1]
  1301. raise
  1302. # EXEC clears any watched keys
  1303. self.watching = False
  1304. if response is None:
  1305. raise WatchError("Watched variable changed.")
  1306. # put any parse errors into the response
  1307. for i, e in errors:
  1308. response.insert(i, e)
  1309. if len(response) != len(commands):
  1310. self.connection.disconnect()
  1311. raise ResponseError(
  1312. "Wrong number of response items from pipeline execution"
  1313. )
  1314. # find any errors in the response and raise if necessary
  1315. if raise_on_error:
  1316. self.raise_first_error(commands, response)
  1317. # We have to run response callbacks manually
  1318. data = []
  1319. for r, cmd in zip(response, commands):
  1320. if not isinstance(r, Exception):
  1321. args, options = cmd
  1322. # Remove keys entry, it needs only for cache.
  1323. options.pop("keys", None)
  1324. command_name = args[0]
  1325. if command_name in self.response_callbacks:
  1326. r = self.response_callbacks[command_name](r, **options)
  1327. data.append(r)
  1328. return data
  1329. def _execute_pipeline(self, connection, commands, raise_on_error):
  1330. # build up all commands into a single request to increase network perf
  1331. all_cmds = connection.pack_commands([args for args, _ in commands])
  1332. connection.send_packed_command(all_cmds)
  1333. response = []
  1334. for args, options in commands:
  1335. try:
  1336. response.append(self.parse_response(connection, args[0], **options))
  1337. except ResponseError as e:
  1338. response.append(e)
  1339. if raise_on_error:
  1340. self.raise_first_error(commands, response)
  1341. return response
  1342. def raise_first_error(self, commands, response):
  1343. for i, r in enumerate(response):
  1344. if isinstance(r, ResponseError):
  1345. self.annotate_exception(r, i + 1, commands[i][0])
  1346. raise r
  1347. def annotate_exception(self, exception, number, command):
  1348. cmd = " ".join(map(safe_str, command))
  1349. msg = (
  1350. f"Command # {number} ({truncate_text(cmd)}) of pipeline "
  1351. f"caused error: {exception.args[0]}"
  1352. )
  1353. exception.args = (msg,) + exception.args[1:]
  1354. def parse_response(self, connection, command_name, **options):
  1355. result = Redis.parse_response(self, connection, command_name, **options)
  1356. if command_name in self.UNWATCH_COMMANDS:
  1357. self.watching = False
  1358. elif command_name == "WATCH":
  1359. self.watching = True
  1360. return result
  1361. def load_scripts(self):
  1362. # make sure all scripts that are about to be run on this pipeline exist
  1363. scripts = list(self.scripts)
  1364. immediate = self.immediate_execute_command
  1365. shas = [s.sha for s in scripts]
  1366. # we can't use the normal script_* methods because they would just
  1367. # get buffered in the pipeline.
  1368. exists = immediate("SCRIPT EXISTS", *shas)
  1369. if not all(exists):
  1370. for s, exist in zip(scripts, exists):
  1371. if not exist:
  1372. s.sha = immediate("SCRIPT LOAD", s.script)
  1373. def _disconnect_raise_on_watching(
  1374. self,
  1375. conn: AbstractConnection,
  1376. error: Exception,
  1377. ) -> None:
  1378. """
  1379. Close the connection, raise an exception if we were watching.
  1380. The supported exceptions are already checked in the
  1381. retry object so we don't need to do it here.
  1382. After we disconnect the connection, it will try to reconnect and
  1383. do a health check as part of the send_command logic(on connection level).
  1384. """
  1385. conn.disconnect()
  1386. # if we were watching a variable, the watch is no longer valid
  1387. # since this connection has died. raise a WatchError, which
  1388. # indicates the user should retry this transaction.
  1389. if self.watching:
  1390. raise WatchError(
  1391. f"A {type(error).__name__} occurred while watching one or more keys"
  1392. )
  1393. def execute(self, raise_on_error: bool = True) -> List[Any]:
  1394. """Execute all the commands in the current pipeline"""
  1395. stack = self.command_stack
  1396. if not stack and not self.watching:
  1397. return []
  1398. if self.scripts:
  1399. self.load_scripts()
  1400. if self.transaction or self.explicit_transaction:
  1401. execute = self._execute_transaction
  1402. else:
  1403. execute = self._execute_pipeline
  1404. conn = self.connection
  1405. if not conn:
  1406. conn = self.connection_pool.get_connection()
  1407. # assign to self.connection so reset() releases the connection
  1408. # back to the pool after we're done
  1409. self.connection = conn
  1410. try:
  1411. return conn.retry.call_with_retry(
  1412. lambda: execute(conn, stack, raise_on_error),
  1413. lambda error: self._disconnect_raise_on_watching(conn, error),
  1414. )
  1415. finally:
  1416. self.reset()
  1417. def discard(self):
  1418. """
  1419. Flushes all previously queued commands
  1420. See: https://redis.io/commands/DISCARD
  1421. """
  1422. self.execute_command("DISCARD")
  1423. def watch(self, *names):
  1424. """Watches the values at keys ``names``"""
  1425. if self.explicit_transaction:
  1426. raise RedisError("Cannot issue a WATCH after a MULTI")
  1427. return self.execute_command("WATCH", *names)
  1428. def unwatch(self) -> bool:
  1429. """Unwatches all previously specified keys"""
  1430. return self.watching and self.execute_command("UNWATCH") or True