client.py 60 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618
  1. import asyncio
  2. import copy
  3. import inspect
  4. import re
  5. import warnings
  6. from typing import (
  7. TYPE_CHECKING,
  8. Any,
  9. AsyncIterator,
  10. Awaitable,
  11. Callable,
  12. Dict,
  13. Iterable,
  14. List,
  15. Mapping,
  16. MutableMapping,
  17. Optional,
  18. Protocol,
  19. Set,
  20. Tuple,
  21. Type,
  22. TypedDict,
  23. TypeVar,
  24. Union,
  25. cast,
  26. )
  27. from redis._parsers.helpers import (
  28. _RedisCallbacks,
  29. _RedisCallbacksRESP2,
  30. _RedisCallbacksRESP3,
  31. bool_ok,
  32. )
  33. from redis.asyncio.connection import (
  34. Connection,
  35. ConnectionPool,
  36. SSLConnection,
  37. UnixDomainSocketConnection,
  38. )
  39. from redis.asyncio.lock import Lock
  40. from redis.asyncio.retry import Retry
  41. from redis.backoff import ExponentialWithJitterBackoff
  42. from redis.client import (
  43. EMPTY_RESPONSE,
  44. NEVER_DECODE,
  45. AbstractRedis,
  46. CaseInsensitiveDict,
  47. )
  48. from redis.commands import (
  49. AsyncCoreCommands,
  50. AsyncRedisModuleCommands,
  51. AsyncSentinelCommands,
  52. list_or_args,
  53. )
  54. from redis.credentials import CredentialProvider
  55. from redis.event import (
  56. AfterPooledConnectionsInstantiationEvent,
  57. AfterPubSubConnectionInstantiationEvent,
  58. AfterSingleConnectionInstantiationEvent,
  59. ClientType,
  60. EventDispatcher,
  61. )
  62. from redis.exceptions import (
  63. ConnectionError,
  64. ExecAbortError,
  65. PubSubError,
  66. RedisError,
  67. ResponseError,
  68. WatchError,
  69. )
  70. from redis.typing import ChannelT, EncodableT, KeyT
  71. from redis.utils import (
  72. SSL_AVAILABLE,
  73. _set_info_logger,
  74. deprecated_args,
  75. deprecated_function,
  76. get_lib_version,
  77. safe_str,
  78. str_if_bytes,
  79. truncate_text,
  80. )
  81. if TYPE_CHECKING and SSL_AVAILABLE:
  82. from ssl import TLSVersion, VerifyMode
  83. else:
  84. TLSVersion = None
  85. VerifyMode = None
  86. PubSubHandler = Callable[[Dict[str, str]], Awaitable[None]]
  87. _KeyT = TypeVar("_KeyT", bound=KeyT)
  88. _ArgT = TypeVar("_ArgT", KeyT, EncodableT)
  89. _RedisT = TypeVar("_RedisT", bound="Redis")
  90. _NormalizeKeysT = TypeVar("_NormalizeKeysT", bound=Mapping[ChannelT, object])
  91. if TYPE_CHECKING:
  92. from redis.commands.core import Script
  93. class ResponseCallbackProtocol(Protocol):
  94. def __call__(self, response: Any, **kwargs): ...
  95. class AsyncResponseCallbackProtocol(Protocol):
  96. async def __call__(self, response: Any, **kwargs): ...
  97. ResponseCallbackT = Union[ResponseCallbackProtocol, AsyncResponseCallbackProtocol]
  98. class Redis(
  99. AbstractRedis, AsyncRedisModuleCommands, AsyncCoreCommands, AsyncSentinelCommands
  100. ):
  101. """
  102. Implementation of the Redis protocol.
  103. This abstract class provides a Python interface to all Redis commands
  104. and an implementation of the Redis protocol.
  105. Pipelines derive from this, implementing how
  106. the commands are sent and received to the Redis server. Based on
  107. configuration, an instance will either use a ConnectionPool, or
  108. Connection object to talk to redis.
  109. """
  110. response_callbacks: MutableMapping[Union[str, bytes], ResponseCallbackT]
  111. @classmethod
  112. def from_url(
  113. cls,
  114. url: str,
  115. single_connection_client: bool = False,
  116. auto_close_connection_pool: Optional[bool] = None,
  117. **kwargs,
  118. ):
  119. """
  120. Return a Redis client object configured from the given URL
  121. For example::
  122. redis://[[username]:[password]]@localhost:6379/0
  123. rediss://[[username]:[password]]@localhost:6379/0
  124. unix://[username@]/path/to/socket.sock?db=0[&password=password]
  125. Three URL schemes are supported:
  126. - `redis://` creates a TCP socket connection. See more at:
  127. <https://www.iana.org/assignments/uri-schemes/prov/redis>
  128. - `rediss://` creates a SSL wrapped TCP socket connection. See more at:
  129. <https://www.iana.org/assignments/uri-schemes/prov/rediss>
  130. - ``unix://``: creates a Unix Domain Socket connection.
  131. The username, password, hostname, path and all querystring values
  132. are passed through urllib.parse.unquote in order to replace any
  133. percent-encoded values with their corresponding characters.
  134. There are several ways to specify a database number. The first value
  135. found will be used:
  136. 1. A ``db`` querystring option, e.g. redis://localhost?db=0
  137. 2. If using the redis:// or rediss:// schemes, the path argument
  138. of the url, e.g. redis://localhost/0
  139. 3. A ``db`` keyword argument to this function.
  140. If none of these options are specified, the default db=0 is used.
  141. All querystring options are cast to their appropriate Python types.
  142. Boolean arguments can be specified with string values "True"/"False"
  143. or "Yes"/"No". Values that cannot be properly cast cause a
  144. ``ValueError`` to be raised. Once parsed, the querystring arguments
  145. and keyword arguments are passed to the ``ConnectionPool``'s
  146. class initializer. In the case of conflicting arguments, querystring
  147. arguments always win.
  148. """
  149. connection_pool = ConnectionPool.from_url(url, **kwargs)
  150. client = cls(
  151. connection_pool=connection_pool,
  152. single_connection_client=single_connection_client,
  153. )
  154. if auto_close_connection_pool is not None:
  155. warnings.warn(
  156. DeprecationWarning(
  157. '"auto_close_connection_pool" is deprecated '
  158. "since version 5.0.1. "
  159. "Please create a ConnectionPool explicitly and "
  160. "provide to the Redis() constructor instead."
  161. )
  162. )
  163. else:
  164. auto_close_connection_pool = True
  165. client.auto_close_connection_pool = auto_close_connection_pool
  166. return client
  167. @classmethod
  168. def from_pool(
  169. cls: Type["Redis"],
  170. connection_pool: ConnectionPool,
  171. ) -> "Redis":
  172. """
  173. Return a Redis client from the given connection pool.
  174. The Redis client will take ownership of the connection pool and
  175. close it when the Redis client is closed.
  176. """
  177. client = cls(
  178. connection_pool=connection_pool,
  179. )
  180. client.auto_close_connection_pool = True
  181. return client
  182. @deprecated_args(
  183. args_to_warn=["retry_on_timeout"],
  184. reason="TimeoutError is included by default.",
  185. version="6.0.0",
  186. )
  187. def __init__(
  188. self,
  189. *,
  190. host: str = "localhost",
  191. port: int = 6379,
  192. db: Union[str, int] = 0,
  193. password: Optional[str] = None,
  194. socket_timeout: Optional[float] = None,
  195. socket_connect_timeout: Optional[float] = None,
  196. socket_keepalive: Optional[bool] = None,
  197. socket_keepalive_options: Optional[Mapping[int, Union[int, bytes]]] = None,
  198. connection_pool: Optional[ConnectionPool] = None,
  199. unix_socket_path: Optional[str] = None,
  200. encoding: str = "utf-8",
  201. encoding_errors: str = "strict",
  202. decode_responses: bool = False,
  203. retry_on_timeout: bool = False,
  204. retry: Retry = Retry(
  205. backoff=ExponentialWithJitterBackoff(base=1, cap=10), retries=3
  206. ),
  207. retry_on_error: Optional[list] = None,
  208. ssl: bool = False,
  209. ssl_keyfile: Optional[str] = None,
  210. ssl_certfile: Optional[str] = None,
  211. ssl_cert_reqs: Union[str, VerifyMode] = "required",
  212. ssl_ca_certs: Optional[str] = None,
  213. ssl_ca_data: Optional[str] = None,
  214. ssl_check_hostname: bool = True,
  215. ssl_min_version: Optional[TLSVersion] = None,
  216. ssl_ciphers: Optional[str] = None,
  217. max_connections: Optional[int] = None,
  218. single_connection_client: bool = False,
  219. health_check_interval: int = 0,
  220. client_name: Optional[str] = None,
  221. lib_name: Optional[str] = "redis-py",
  222. lib_version: Optional[str] = get_lib_version(),
  223. username: Optional[str] = None,
  224. auto_close_connection_pool: Optional[bool] = None,
  225. redis_connect_func=None,
  226. credential_provider: Optional[CredentialProvider] = None,
  227. protocol: Optional[int] = 2,
  228. event_dispatcher: Optional[EventDispatcher] = None,
  229. ):
  230. """
  231. Initialize a new Redis client.
  232. To specify a retry policy for specific errors, you have two options:
  233. 1. Set the `retry_on_error` to a list of the error/s to retry on, and
  234. you can also set `retry` to a valid `Retry` object(in case the default
  235. one is not appropriate) - with this approach the retries will be triggered
  236. on the default errors specified in the Retry object enriched with the
  237. errors specified in `retry_on_error`.
  238. 2. Define a `Retry` object with configured 'supported_errors' and set
  239. it to the `retry` parameter - with this approach you completely redefine
  240. the errors on which retries will happen.
  241. `retry_on_timeout` is deprecated - please include the TimeoutError
  242. either in the Retry object or in the `retry_on_error` list.
  243. When 'connection_pool' is provided - the retry configuration of the
  244. provided pool will be used.
  245. """
  246. kwargs: Dict[str, Any]
  247. if event_dispatcher is None:
  248. self._event_dispatcher = EventDispatcher()
  249. else:
  250. self._event_dispatcher = event_dispatcher
  251. # auto_close_connection_pool only has an effect if connection_pool is
  252. # None. It is assumed that if connection_pool is not None, the user
  253. # wants to manage the connection pool themselves.
  254. if auto_close_connection_pool is not None:
  255. warnings.warn(
  256. DeprecationWarning(
  257. '"auto_close_connection_pool" is deprecated '
  258. "since version 5.0.1. "
  259. "Please create a ConnectionPool explicitly and "
  260. "provide to the Redis() constructor instead."
  261. )
  262. )
  263. else:
  264. auto_close_connection_pool = True
  265. if not connection_pool:
  266. # Create internal connection pool, expected to be closed by Redis instance
  267. if not retry_on_error:
  268. retry_on_error = []
  269. kwargs = {
  270. "db": db,
  271. "username": username,
  272. "password": password,
  273. "credential_provider": credential_provider,
  274. "socket_timeout": socket_timeout,
  275. "encoding": encoding,
  276. "encoding_errors": encoding_errors,
  277. "decode_responses": decode_responses,
  278. "retry_on_error": retry_on_error,
  279. "retry": copy.deepcopy(retry),
  280. "max_connections": max_connections,
  281. "health_check_interval": health_check_interval,
  282. "client_name": client_name,
  283. "lib_name": lib_name,
  284. "lib_version": lib_version,
  285. "redis_connect_func": redis_connect_func,
  286. "protocol": protocol,
  287. }
  288. # based on input, setup appropriate connection args
  289. if unix_socket_path is not None:
  290. kwargs.update(
  291. {
  292. "path": unix_socket_path,
  293. "connection_class": UnixDomainSocketConnection,
  294. }
  295. )
  296. else:
  297. # TCP specific options
  298. kwargs.update(
  299. {
  300. "host": host,
  301. "port": port,
  302. "socket_connect_timeout": socket_connect_timeout,
  303. "socket_keepalive": socket_keepalive,
  304. "socket_keepalive_options": socket_keepalive_options,
  305. }
  306. )
  307. if ssl:
  308. kwargs.update(
  309. {
  310. "connection_class": SSLConnection,
  311. "ssl_keyfile": ssl_keyfile,
  312. "ssl_certfile": ssl_certfile,
  313. "ssl_cert_reqs": ssl_cert_reqs,
  314. "ssl_ca_certs": ssl_ca_certs,
  315. "ssl_ca_data": ssl_ca_data,
  316. "ssl_check_hostname": ssl_check_hostname,
  317. "ssl_min_version": ssl_min_version,
  318. "ssl_ciphers": ssl_ciphers,
  319. }
  320. )
  321. # This arg only used if no pool is passed in
  322. self.auto_close_connection_pool = auto_close_connection_pool
  323. connection_pool = ConnectionPool(**kwargs)
  324. self._event_dispatcher.dispatch(
  325. AfterPooledConnectionsInstantiationEvent(
  326. [connection_pool], ClientType.ASYNC, credential_provider
  327. )
  328. )
  329. else:
  330. # If a pool is passed in, do not close it
  331. self.auto_close_connection_pool = False
  332. self._event_dispatcher.dispatch(
  333. AfterPooledConnectionsInstantiationEvent(
  334. [connection_pool], ClientType.ASYNC, credential_provider
  335. )
  336. )
  337. self.connection_pool = connection_pool
  338. self.single_connection_client = single_connection_client
  339. self.connection: Optional[Connection] = None
  340. self.response_callbacks = CaseInsensitiveDict(_RedisCallbacks)
  341. if self.connection_pool.connection_kwargs.get("protocol") in ["3", 3]:
  342. self.response_callbacks.update(_RedisCallbacksRESP3)
  343. else:
  344. self.response_callbacks.update(_RedisCallbacksRESP2)
  345. # If using a single connection client, we need to lock creation-of and use-of
  346. # the client in order to avoid race conditions such as using asyncio.gather
  347. # on a set of redis commands
  348. self._single_conn_lock = asyncio.Lock()
  349. def __repr__(self):
  350. return (
  351. f"<{self.__class__.__module__}.{self.__class__.__name__}"
  352. f"({self.connection_pool!r})>"
  353. )
  354. def __await__(self):
  355. return self.initialize().__await__()
  356. async def initialize(self: _RedisT) -> _RedisT:
  357. if self.single_connection_client:
  358. async with self._single_conn_lock:
  359. if self.connection is None:
  360. self.connection = await self.connection_pool.get_connection()
  361. self._event_dispatcher.dispatch(
  362. AfterSingleConnectionInstantiationEvent(
  363. self.connection, ClientType.ASYNC, self._single_conn_lock
  364. )
  365. )
  366. return self
  367. def set_response_callback(self, command: str, callback: ResponseCallbackT):
  368. """Set a custom Response Callback"""
  369. self.response_callbacks[command] = callback
  370. def get_encoder(self):
  371. """Get the connection pool's encoder"""
  372. return self.connection_pool.get_encoder()
  373. def get_connection_kwargs(self):
  374. """Get the connection's key-word arguments"""
  375. return self.connection_pool.connection_kwargs
  376. def get_retry(self) -> Optional[Retry]:
  377. return self.get_connection_kwargs().get("retry")
  378. def set_retry(self, retry: Retry) -> None:
  379. self.get_connection_kwargs().update({"retry": retry})
  380. self.connection_pool.set_retry(retry)
  381. def load_external_module(self, funcname, func):
  382. """
  383. This function can be used to add externally defined redis modules,
  384. and their namespaces to the redis client.
  385. funcname - A string containing the name of the function to create
  386. func - The function, being added to this class.
  387. ex: Assume that one has a custom redis module named foomod that
  388. creates command named 'foo.dothing' and 'foo.anotherthing' in redis.
  389. To load function functions into this namespace:
  390. from redis import Redis
  391. from foomodule import F
  392. r = Redis()
  393. r.load_external_module("foo", F)
  394. r.foo().dothing('your', 'arguments')
  395. For a concrete example see the reimport of the redisjson module in
  396. tests/test_connection.py::test_loading_external_modules
  397. """
  398. setattr(self, funcname, func)
  399. def pipeline(
  400. self, transaction: bool = True, shard_hint: Optional[str] = None
  401. ) -> "Pipeline":
  402. """
  403. Return a new pipeline object that can queue multiple commands for
  404. later execution. ``transaction`` indicates whether all commands
  405. should be executed atomically. Apart from making a group of operations
  406. atomic, pipelines are useful for reducing the back-and-forth overhead
  407. between the client and server.
  408. """
  409. return Pipeline(
  410. self.connection_pool, self.response_callbacks, transaction, shard_hint
  411. )
  412. async def transaction(
  413. self,
  414. func: Callable[["Pipeline"], Union[Any, Awaitable[Any]]],
  415. *watches: KeyT,
  416. shard_hint: Optional[str] = None,
  417. value_from_callable: bool = False,
  418. watch_delay: Optional[float] = None,
  419. ):
  420. """
  421. Convenience method for executing the callable `func` as a transaction
  422. while watching all keys specified in `watches`. The 'func' callable
  423. should expect a single argument which is a Pipeline object.
  424. """
  425. pipe: Pipeline
  426. async with self.pipeline(True, shard_hint) as pipe:
  427. while True:
  428. try:
  429. if watches:
  430. await pipe.watch(*watches)
  431. func_value = func(pipe)
  432. if inspect.isawaitable(func_value):
  433. func_value = await func_value
  434. exec_value = await pipe.execute()
  435. return func_value if value_from_callable else exec_value
  436. except WatchError:
  437. if watch_delay is not None and watch_delay > 0:
  438. await asyncio.sleep(watch_delay)
  439. continue
  440. def lock(
  441. self,
  442. name: KeyT,
  443. timeout: Optional[float] = None,
  444. sleep: float = 0.1,
  445. blocking: bool = True,
  446. blocking_timeout: Optional[float] = None,
  447. lock_class: Optional[Type[Lock]] = None,
  448. thread_local: bool = True,
  449. raise_on_release_error: bool = True,
  450. ) -> Lock:
  451. """
  452. Return a new Lock object using key ``name`` that mimics
  453. the behavior of threading.Lock.
  454. If specified, ``timeout`` indicates a maximum life for the lock.
  455. By default, it will remain locked until release() is called.
  456. ``sleep`` indicates the amount of time to sleep per loop iteration
  457. when the lock is in blocking mode and another client is currently
  458. holding the lock.
  459. ``blocking`` indicates whether calling ``acquire`` should block until
  460. the lock has been acquired or to fail immediately, causing ``acquire``
  461. to return False and the lock not being acquired. Defaults to True.
  462. Note this value can be overridden by passing a ``blocking``
  463. argument to ``acquire``.
  464. ``blocking_timeout`` indicates the maximum amount of time in seconds to
  465. spend trying to acquire the lock. A value of ``None`` indicates
  466. continue trying forever. ``blocking_timeout`` can be specified as a
  467. float or integer, both representing the number of seconds to wait.
  468. ``lock_class`` forces the specified lock implementation. Note that as
  469. of redis-py 3.0, the only lock class we implement is ``Lock`` (which is
  470. a Lua-based lock). So, it's unlikely you'll need this parameter, unless
  471. you have created your own custom lock class.
  472. ``thread_local`` indicates whether the lock token is placed in
  473. thread-local storage. By default, the token is placed in thread local
  474. storage so that a thread only sees its token, not a token set by
  475. another thread. Consider the following timeline:
  476. time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds.
  477. thread-1 sets the token to "abc"
  478. time: 1, thread-2 blocks trying to acquire `my-lock` using the
  479. Lock instance.
  480. time: 5, thread-1 has not yet completed. redis expires the lock
  481. key.
  482. time: 5, thread-2 acquired `my-lock` now that it's available.
  483. thread-2 sets the token to "xyz"
  484. time: 6, thread-1 finishes its work and calls release(). if the
  485. token is *not* stored in thread local storage, then
  486. thread-1 would see the token value as "xyz" and would be
  487. able to successfully release the thread-2's lock.
  488. ``raise_on_release_error`` indicates whether to raise an exception when
  489. the lock is no longer owned when exiting the context manager. By default,
  490. this is True, meaning an exception will be raised. If False, the warning
  491. will be logged and the exception will be suppressed.
  492. In some use cases it's necessary to disable thread local storage. For
  493. example, if you have code where one thread acquires a lock and passes
  494. that lock instance to a worker thread to release later. If thread
  495. local storage isn't disabled in this case, the worker thread won't see
  496. the token set by the thread that acquired the lock. Our assumption
  497. is that these cases aren't common and as such default to using
  498. thread local storage."""
  499. if lock_class is None:
  500. lock_class = Lock
  501. return lock_class(
  502. self,
  503. name,
  504. timeout=timeout,
  505. sleep=sleep,
  506. blocking=blocking,
  507. blocking_timeout=blocking_timeout,
  508. thread_local=thread_local,
  509. raise_on_release_error=raise_on_release_error,
  510. )
  511. def pubsub(self, **kwargs) -> "PubSub":
  512. """
  513. Return a Publish/Subscribe object. With this object, you can
  514. subscribe to channels and listen for messages that get published to
  515. them.
  516. """
  517. return PubSub(
  518. self.connection_pool, event_dispatcher=self._event_dispatcher, **kwargs
  519. )
  520. def monitor(self) -> "Monitor":
  521. return Monitor(self.connection_pool)
  522. def client(self) -> "Redis":
  523. return self.__class__(
  524. connection_pool=self.connection_pool, single_connection_client=True
  525. )
  526. async def __aenter__(self: _RedisT) -> _RedisT:
  527. return await self.initialize()
  528. async def __aexit__(self, exc_type, exc_value, traceback):
  529. await self.aclose()
  530. _DEL_MESSAGE = "Unclosed Redis client"
  531. # passing _warnings and _grl as argument default since they may be gone
  532. # by the time __del__ is called at shutdown
  533. def __del__(
  534. self,
  535. _warn: Any = warnings.warn,
  536. _grl: Any = asyncio.get_running_loop,
  537. ) -> None:
  538. if hasattr(self, "connection") and (self.connection is not None):
  539. _warn(f"Unclosed client session {self!r}", ResourceWarning, source=self)
  540. try:
  541. context = {"client": self, "message": self._DEL_MESSAGE}
  542. _grl().call_exception_handler(context)
  543. except RuntimeError:
  544. pass
  545. self.connection._close()
  546. async def aclose(self, close_connection_pool: Optional[bool] = None) -> None:
  547. """
  548. Closes Redis client connection
  549. Args:
  550. close_connection_pool:
  551. decides whether to close the connection pool used by this Redis client,
  552. overriding Redis.auto_close_connection_pool.
  553. By default, let Redis.auto_close_connection_pool decide
  554. whether to close the connection pool.
  555. """
  556. conn = self.connection
  557. if conn:
  558. self.connection = None
  559. await self.connection_pool.release(conn)
  560. if close_connection_pool or (
  561. close_connection_pool is None and self.auto_close_connection_pool
  562. ):
  563. await self.connection_pool.disconnect()
  564. @deprecated_function(version="5.0.1", reason="Use aclose() instead", name="close")
  565. async def close(self, close_connection_pool: Optional[bool] = None) -> None:
  566. """
  567. Alias for aclose(), for backwards compatibility
  568. """
  569. await self.aclose(close_connection_pool)
  570. async def _send_command_parse_response(self, conn, command_name, *args, **options):
  571. """
  572. Send a command and parse the response
  573. """
  574. await conn.send_command(*args)
  575. return await self.parse_response(conn, command_name, **options)
  576. async def _close_connection(self, conn: Connection):
  577. """
  578. Close the connection before retrying.
  579. The supported exceptions are already checked in the
  580. retry object so we don't need to do it here.
  581. After we disconnect the connection, it will try to reconnect and
  582. do a health check as part of the send_command logic(on connection level).
  583. """
  584. await conn.disconnect()
  585. # COMMAND EXECUTION AND PROTOCOL PARSING
  586. async def execute_command(self, *args, **options):
  587. """Execute a command and return a parsed response"""
  588. await self.initialize()
  589. pool = self.connection_pool
  590. command_name = args[0]
  591. conn = self.connection or await pool.get_connection()
  592. if self.single_connection_client:
  593. await self._single_conn_lock.acquire()
  594. try:
  595. return await conn.retry.call_with_retry(
  596. lambda: self._send_command_parse_response(
  597. conn, command_name, *args, **options
  598. ),
  599. lambda _: self._close_connection(conn),
  600. )
  601. finally:
  602. if self.single_connection_client:
  603. self._single_conn_lock.release()
  604. if not self.connection:
  605. await pool.release(conn)
  606. async def parse_response(
  607. self, connection: Connection, command_name: Union[str, bytes], **options
  608. ):
  609. """Parses a response from the Redis server"""
  610. try:
  611. if NEVER_DECODE in options:
  612. response = await connection.read_response(disable_decoding=True)
  613. options.pop(NEVER_DECODE)
  614. else:
  615. response = await connection.read_response()
  616. except ResponseError:
  617. if EMPTY_RESPONSE in options:
  618. return options[EMPTY_RESPONSE]
  619. raise
  620. if EMPTY_RESPONSE in options:
  621. options.pop(EMPTY_RESPONSE)
  622. # Remove keys entry, it needs only for cache.
  623. options.pop("keys", None)
  624. if command_name in self.response_callbacks:
  625. # Mypy bug: https://github.com/python/mypy/issues/10977
  626. command_name = cast(str, command_name)
  627. retval = self.response_callbacks[command_name](response, **options)
  628. return await retval if inspect.isawaitable(retval) else retval
  629. return response
  630. StrictRedis = Redis
  631. class MonitorCommandInfo(TypedDict):
  632. time: float
  633. db: int
  634. client_address: str
  635. client_port: str
  636. client_type: str
  637. command: str
  638. class Monitor:
  639. """
  640. Monitor is useful for handling the MONITOR command to the redis server.
  641. next_command() method returns one command from monitor
  642. listen() method yields commands from monitor.
  643. """
  644. monitor_re = re.compile(r"\[(\d+) (.*?)\] (.*)")
  645. command_re = re.compile(r'"(.*?)(?<!\\)"')
  646. def __init__(self, connection_pool: ConnectionPool):
  647. self.connection_pool = connection_pool
  648. self.connection: Optional[Connection] = None
  649. async def connect(self):
  650. if self.connection is None:
  651. self.connection = await self.connection_pool.get_connection()
  652. async def __aenter__(self):
  653. await self.connect()
  654. await self.connection.send_command("MONITOR")
  655. # check that monitor returns 'OK', but don't return it to user
  656. response = await self.connection.read_response()
  657. if not bool_ok(response):
  658. raise RedisError(f"MONITOR failed: {response}")
  659. return self
  660. async def __aexit__(self, *args):
  661. await self.connection.disconnect()
  662. await self.connection_pool.release(self.connection)
  663. async def next_command(self) -> MonitorCommandInfo:
  664. """Parse the response from a monitor command"""
  665. await self.connect()
  666. response = await self.connection.read_response()
  667. if isinstance(response, bytes):
  668. response = self.connection.encoder.decode(response, force=True)
  669. command_time, command_data = response.split(" ", 1)
  670. m = self.monitor_re.match(command_data)
  671. db_id, client_info, command = m.groups()
  672. command = " ".join(self.command_re.findall(command))
  673. # Redis escapes double quotes because each piece of the command
  674. # string is surrounded by double quotes. We don't have that
  675. # requirement so remove the escaping and leave the quote.
  676. command = command.replace('\\"', '"')
  677. if client_info == "lua":
  678. client_address = "lua"
  679. client_port = ""
  680. client_type = "lua"
  681. elif client_info.startswith("unix"):
  682. client_address = "unix"
  683. client_port = client_info[5:]
  684. client_type = "unix"
  685. else:
  686. # use rsplit as ipv6 addresses contain colons
  687. client_address, client_port = client_info.rsplit(":", 1)
  688. client_type = "tcp"
  689. return {
  690. "time": float(command_time),
  691. "db": int(db_id),
  692. "client_address": client_address,
  693. "client_port": client_port,
  694. "client_type": client_type,
  695. "command": command,
  696. }
  697. async def listen(self) -> AsyncIterator[MonitorCommandInfo]:
  698. """Listen for commands coming to the server."""
  699. while True:
  700. yield await self.next_command()
  701. class PubSub:
  702. """
  703. PubSub provides publish, subscribe and listen support to Redis channels.
  704. After subscribing to one or more channels, the listen() method will block
  705. until a message arrives on one of the subscribed channels. That message
  706. will be returned and it's safe to start listening again.
  707. """
  708. PUBLISH_MESSAGE_TYPES = ("message", "pmessage")
  709. UNSUBSCRIBE_MESSAGE_TYPES = ("unsubscribe", "punsubscribe")
  710. HEALTH_CHECK_MESSAGE = "redis-py-health-check"
  711. def __init__(
  712. self,
  713. connection_pool: ConnectionPool,
  714. shard_hint: Optional[str] = None,
  715. ignore_subscribe_messages: bool = False,
  716. encoder=None,
  717. push_handler_func: Optional[Callable] = None,
  718. event_dispatcher: Optional["EventDispatcher"] = None,
  719. ):
  720. if event_dispatcher is None:
  721. self._event_dispatcher = EventDispatcher()
  722. else:
  723. self._event_dispatcher = event_dispatcher
  724. self.connection_pool = connection_pool
  725. self.shard_hint = shard_hint
  726. self.ignore_subscribe_messages = ignore_subscribe_messages
  727. self.connection = None
  728. # we need to know the encoding options for this connection in order
  729. # to lookup channel and pattern names for callback handlers.
  730. self.encoder = encoder
  731. self.push_handler_func = push_handler_func
  732. if self.encoder is None:
  733. self.encoder = self.connection_pool.get_encoder()
  734. if self.encoder.decode_responses:
  735. self.health_check_response = [
  736. ["pong", self.HEALTH_CHECK_MESSAGE],
  737. self.HEALTH_CHECK_MESSAGE,
  738. ]
  739. else:
  740. self.health_check_response = [
  741. [b"pong", self.encoder.encode(self.HEALTH_CHECK_MESSAGE)],
  742. self.encoder.encode(self.HEALTH_CHECK_MESSAGE),
  743. ]
  744. if self.push_handler_func is None:
  745. _set_info_logger()
  746. self.channels = {}
  747. self.pending_unsubscribe_channels = set()
  748. self.patterns = {}
  749. self.pending_unsubscribe_patterns = set()
  750. self._lock = asyncio.Lock()
  751. async def __aenter__(self):
  752. return self
  753. async def __aexit__(self, exc_type, exc_value, traceback):
  754. await self.aclose()
  755. def __del__(self):
  756. if self.connection:
  757. self.connection.deregister_connect_callback(self.on_connect)
  758. async def aclose(self):
  759. # In case a connection property does not yet exist
  760. # (due to a crash earlier in the Redis() constructor), return
  761. # immediately as there is nothing to clean-up.
  762. if not hasattr(self, "connection"):
  763. return
  764. async with self._lock:
  765. if self.connection:
  766. await self.connection.disconnect()
  767. self.connection.deregister_connect_callback(self.on_connect)
  768. await self.connection_pool.release(self.connection)
  769. self.connection = None
  770. self.channels = {}
  771. self.pending_unsubscribe_channels = set()
  772. self.patterns = {}
  773. self.pending_unsubscribe_patterns = set()
  774. @deprecated_function(version="5.0.1", reason="Use aclose() instead", name="close")
  775. async def close(self) -> None:
  776. """Alias for aclose(), for backwards compatibility"""
  777. await self.aclose()
  778. @deprecated_function(version="5.0.1", reason="Use aclose() instead", name="reset")
  779. async def reset(self) -> None:
  780. """Alias for aclose(), for backwards compatibility"""
  781. await self.aclose()
  782. async def on_connect(self, connection: Connection):
  783. """Re-subscribe to any channels and patterns previously subscribed to"""
  784. # NOTE: for python3, we can't pass bytestrings as keyword arguments
  785. # so we need to decode channel/pattern names back to unicode strings
  786. # before passing them to [p]subscribe.
  787. self.pending_unsubscribe_channels.clear()
  788. self.pending_unsubscribe_patterns.clear()
  789. if self.channels:
  790. channels = {}
  791. for k, v in self.channels.items():
  792. channels[self.encoder.decode(k, force=True)] = v
  793. await self.subscribe(**channels)
  794. if self.patterns:
  795. patterns = {}
  796. for k, v in self.patterns.items():
  797. patterns[self.encoder.decode(k, force=True)] = v
  798. await self.psubscribe(**patterns)
  799. @property
  800. def subscribed(self):
  801. """Indicates if there are subscriptions to any channels or patterns"""
  802. return bool(self.channels or self.patterns)
  803. async def execute_command(self, *args: EncodableT):
  804. """Execute a publish/subscribe command"""
  805. # NOTE: don't parse the response in this function -- it could pull a
  806. # legitimate message off the stack if the connection is already
  807. # subscribed to one or more channels
  808. await self.connect()
  809. connection = self.connection
  810. kwargs = {"check_health": not self.subscribed}
  811. await self._execute(connection, connection.send_command, *args, **kwargs)
  812. async def connect(self):
  813. """
  814. Ensure that the PubSub is connected
  815. """
  816. if self.connection is None:
  817. self.connection = await self.connection_pool.get_connection()
  818. # register a callback that re-subscribes to any channels we
  819. # were listening to when we were disconnected
  820. self.connection.register_connect_callback(self.on_connect)
  821. else:
  822. await self.connection.connect()
  823. if self.push_handler_func is not None:
  824. self.connection._parser.set_pubsub_push_handler(self.push_handler_func)
  825. self._event_dispatcher.dispatch(
  826. AfterPubSubConnectionInstantiationEvent(
  827. self.connection, self.connection_pool, ClientType.ASYNC, self._lock
  828. )
  829. )
  830. async def _reconnect(self, conn):
  831. """
  832. Try to reconnect
  833. """
  834. await conn.disconnect()
  835. await conn.connect()
  836. async def _execute(self, conn, command, *args, **kwargs):
  837. """
  838. Connect manually upon disconnection. If the Redis server is down,
  839. this will fail and raise a ConnectionError as desired.
  840. After reconnection, the ``on_connect`` callback should have been
  841. called by the # connection to resubscribe us to any channels and
  842. patterns we were previously listening to
  843. """
  844. return await conn.retry.call_with_retry(
  845. lambda: command(*args, **kwargs),
  846. lambda _: self._reconnect(conn),
  847. )
  848. async def parse_response(self, block: bool = True, timeout: float = 0):
  849. """Parse the response from a publish/subscribe command"""
  850. conn = self.connection
  851. if conn is None:
  852. raise RuntimeError(
  853. "pubsub connection not set: "
  854. "did you forget to call subscribe() or psubscribe()?"
  855. )
  856. await self.check_health()
  857. if not conn.is_connected:
  858. await conn.connect()
  859. read_timeout = None if block else timeout
  860. response = await self._execute(
  861. conn,
  862. conn.read_response,
  863. timeout=read_timeout,
  864. disconnect_on_error=False,
  865. push_request=True,
  866. )
  867. if conn.health_check_interval and response in self.health_check_response:
  868. # ignore the health check message as user might not expect it
  869. return None
  870. return response
  871. async def check_health(self):
  872. conn = self.connection
  873. if conn is None:
  874. raise RuntimeError(
  875. "pubsub connection not set: "
  876. "did you forget to call subscribe() or psubscribe()?"
  877. )
  878. if (
  879. conn.health_check_interval
  880. and asyncio.get_running_loop().time() > conn.next_health_check
  881. ):
  882. await conn.send_command(
  883. "PING", self.HEALTH_CHECK_MESSAGE, check_health=False
  884. )
  885. def _normalize_keys(self, data: _NormalizeKeysT) -> _NormalizeKeysT:
  886. """
  887. normalize channel/pattern names to be either bytes or strings
  888. based on whether responses are automatically decoded. this saves us
  889. from coercing the value for each message coming in.
  890. """
  891. encode = self.encoder.encode
  892. decode = self.encoder.decode
  893. return {decode(encode(k)): v for k, v in data.items()} # type: ignore[return-value] # noqa: E501
  894. async def psubscribe(self, *args: ChannelT, **kwargs: PubSubHandler):
  895. """
  896. Subscribe to channel patterns. Patterns supplied as keyword arguments
  897. expect a pattern name as the key and a callable as the value. A
  898. pattern's callable will be invoked automatically when a message is
  899. received on that pattern rather than producing a message via
  900. ``listen()``.
  901. """
  902. parsed_args = list_or_args((args[0],), args[1:]) if args else args
  903. new_patterns: Dict[ChannelT, PubSubHandler] = dict.fromkeys(parsed_args)
  904. # Mypy bug: https://github.com/python/mypy/issues/10970
  905. new_patterns.update(kwargs) # type: ignore[arg-type]
  906. ret_val = await self.execute_command("PSUBSCRIBE", *new_patterns.keys())
  907. # update the patterns dict AFTER we send the command. we don't want to
  908. # subscribe twice to these patterns, once for the command and again
  909. # for the reconnection.
  910. new_patterns = self._normalize_keys(new_patterns)
  911. self.patterns.update(new_patterns)
  912. self.pending_unsubscribe_patterns.difference_update(new_patterns)
  913. return ret_val
  914. def punsubscribe(self, *args: ChannelT) -> Awaitable:
  915. """
  916. Unsubscribe from the supplied patterns. If empty, unsubscribe from
  917. all patterns.
  918. """
  919. patterns: Iterable[ChannelT]
  920. if args:
  921. parsed_args = list_or_args((args[0],), args[1:])
  922. patterns = self._normalize_keys(dict.fromkeys(parsed_args)).keys()
  923. else:
  924. parsed_args = []
  925. patterns = self.patterns
  926. self.pending_unsubscribe_patterns.update(patterns)
  927. return self.execute_command("PUNSUBSCRIBE", *parsed_args)
  928. async def subscribe(self, *args: ChannelT, **kwargs: Callable):
  929. """
  930. Subscribe to channels. Channels supplied as keyword arguments expect
  931. a channel name as the key and a callable as the value. A channel's
  932. callable will be invoked automatically when a message is received on
  933. that channel rather than producing a message via ``listen()`` or
  934. ``get_message()``.
  935. """
  936. parsed_args = list_or_args((args[0],), args[1:]) if args else ()
  937. new_channels = dict.fromkeys(parsed_args)
  938. # Mypy bug: https://github.com/python/mypy/issues/10970
  939. new_channels.update(kwargs) # type: ignore[arg-type]
  940. ret_val = await self.execute_command("SUBSCRIBE", *new_channels.keys())
  941. # update the channels dict AFTER we send the command. we don't want to
  942. # subscribe twice to these channels, once for the command and again
  943. # for the reconnection.
  944. new_channels = self._normalize_keys(new_channels)
  945. self.channels.update(new_channels)
  946. self.pending_unsubscribe_channels.difference_update(new_channels)
  947. return ret_val
  948. def unsubscribe(self, *args) -> Awaitable:
  949. """
  950. Unsubscribe from the supplied channels. If empty, unsubscribe from
  951. all channels
  952. """
  953. if args:
  954. parsed_args = list_or_args(args[0], args[1:])
  955. channels = self._normalize_keys(dict.fromkeys(parsed_args))
  956. else:
  957. parsed_args = []
  958. channels = self.channels
  959. self.pending_unsubscribe_channels.update(channels)
  960. return self.execute_command("UNSUBSCRIBE", *parsed_args)
  961. async def listen(self) -> AsyncIterator:
  962. """Listen for messages on channels this client has been subscribed to"""
  963. while self.subscribed:
  964. response = await self.handle_message(await self.parse_response(block=True))
  965. if response is not None:
  966. yield response
  967. async def get_message(
  968. self, ignore_subscribe_messages: bool = False, timeout: Optional[float] = 0.0
  969. ):
  970. """
  971. Get the next message if one is available, otherwise None.
  972. If timeout is specified, the system will wait for `timeout` seconds
  973. before returning. Timeout should be specified as a floating point
  974. number or None to wait indefinitely.
  975. """
  976. response = await self.parse_response(block=(timeout is None), timeout=timeout)
  977. if response:
  978. return await self.handle_message(response, ignore_subscribe_messages)
  979. return None
  980. def ping(self, message=None) -> Awaitable:
  981. """
  982. Ping the Redis server
  983. """
  984. args = ["PING", message] if message is not None else ["PING"]
  985. return self.execute_command(*args)
  986. async def handle_message(self, response, ignore_subscribe_messages=False):
  987. """
  988. Parses a pub/sub message. If the channel or pattern was subscribed to
  989. with a message handler, the handler is invoked instead of a parsed
  990. message being returned.
  991. """
  992. if response is None:
  993. return None
  994. if isinstance(response, bytes):
  995. response = [b"pong", response] if response != b"PONG" else [b"pong", b""]
  996. message_type = str_if_bytes(response[0])
  997. if message_type == "pmessage":
  998. message = {
  999. "type": message_type,
  1000. "pattern": response[1],
  1001. "channel": response[2],
  1002. "data": response[3],
  1003. }
  1004. elif message_type == "pong":
  1005. message = {
  1006. "type": message_type,
  1007. "pattern": None,
  1008. "channel": None,
  1009. "data": response[1],
  1010. }
  1011. else:
  1012. message = {
  1013. "type": message_type,
  1014. "pattern": None,
  1015. "channel": response[1],
  1016. "data": response[2],
  1017. }
  1018. # if this is an unsubscribe message, remove it from memory
  1019. if message_type in self.UNSUBSCRIBE_MESSAGE_TYPES:
  1020. if message_type == "punsubscribe":
  1021. pattern = response[1]
  1022. if pattern in self.pending_unsubscribe_patterns:
  1023. self.pending_unsubscribe_patterns.remove(pattern)
  1024. self.patterns.pop(pattern, None)
  1025. else:
  1026. channel = response[1]
  1027. if channel in self.pending_unsubscribe_channels:
  1028. self.pending_unsubscribe_channels.remove(channel)
  1029. self.channels.pop(channel, None)
  1030. if message_type in self.PUBLISH_MESSAGE_TYPES:
  1031. # if there's a message handler, invoke it
  1032. if message_type == "pmessage":
  1033. handler = self.patterns.get(message["pattern"], None)
  1034. else:
  1035. handler = self.channels.get(message["channel"], None)
  1036. if handler:
  1037. if inspect.iscoroutinefunction(handler):
  1038. await handler(message)
  1039. else:
  1040. handler(message)
  1041. return None
  1042. elif message_type != "pong":
  1043. # this is a subscribe/unsubscribe message. ignore if we don't
  1044. # want them
  1045. if ignore_subscribe_messages or self.ignore_subscribe_messages:
  1046. return None
  1047. return message
  1048. async def run(
  1049. self,
  1050. *,
  1051. exception_handler: Optional["PSWorkerThreadExcHandlerT"] = None,
  1052. poll_timeout: float = 1.0,
  1053. ) -> None:
  1054. """Process pub/sub messages using registered callbacks.
  1055. This is the equivalent of :py:meth:`redis.PubSub.run_in_thread` in
  1056. redis-py, but it is a coroutine. To launch it as a separate task, use
  1057. ``asyncio.create_task``:
  1058. >>> task = asyncio.create_task(pubsub.run())
  1059. To shut it down, use asyncio cancellation:
  1060. >>> task.cancel()
  1061. >>> await task
  1062. """
  1063. for channel, handler in self.channels.items():
  1064. if handler is None:
  1065. raise PubSubError(f"Channel: '{channel}' has no handler registered")
  1066. for pattern, handler in self.patterns.items():
  1067. if handler is None:
  1068. raise PubSubError(f"Pattern: '{pattern}' has no handler registered")
  1069. await self.connect()
  1070. while True:
  1071. try:
  1072. await self.get_message(
  1073. ignore_subscribe_messages=True, timeout=poll_timeout
  1074. )
  1075. except asyncio.CancelledError:
  1076. raise
  1077. except BaseException as e:
  1078. if exception_handler is None:
  1079. raise
  1080. res = exception_handler(e, self)
  1081. if inspect.isawaitable(res):
  1082. await res
  1083. # Ensure that other tasks on the event loop get a chance to run
  1084. # if we didn't have to block for I/O anywhere.
  1085. await asyncio.sleep(0)
  1086. class PubsubWorkerExceptionHandler(Protocol):
  1087. def __call__(self, e: BaseException, pubsub: PubSub): ...
  1088. class AsyncPubsubWorkerExceptionHandler(Protocol):
  1089. async def __call__(self, e: BaseException, pubsub: PubSub): ...
  1090. PSWorkerThreadExcHandlerT = Union[
  1091. PubsubWorkerExceptionHandler, AsyncPubsubWorkerExceptionHandler
  1092. ]
  1093. CommandT = Tuple[Tuple[Union[str, bytes], ...], Mapping[str, Any]]
  1094. CommandStackT = List[CommandT]
  1095. class Pipeline(Redis): # lgtm [py/init-calls-subclass]
  1096. """
  1097. Pipelines provide a way to transmit multiple commands to the Redis server
  1098. in one transmission. This is convenient for batch processing, such as
  1099. saving all the values in a list to Redis.
  1100. All commands executed within a pipeline(when running in transactional mode,
  1101. which is the default behavior) are wrapped with MULTI and EXEC
  1102. calls. This guarantees all commands executed in the pipeline will be
  1103. executed atomically.
  1104. Any command raising an exception does *not* halt the execution of
  1105. subsequent commands in the pipeline. Instead, the exception is caught
  1106. and its instance is placed into the response list returned by execute().
  1107. Code iterating over the response list should be able to deal with an
  1108. instance of an exception as a potential value. In general, these will be
  1109. ResponseError exceptions, such as those raised when issuing a command
  1110. on a key of a different datatype.
  1111. """
  1112. UNWATCH_COMMANDS = {"DISCARD", "EXEC", "UNWATCH"}
  1113. def __init__(
  1114. self,
  1115. connection_pool: ConnectionPool,
  1116. response_callbacks: MutableMapping[Union[str, bytes], ResponseCallbackT],
  1117. transaction: bool,
  1118. shard_hint: Optional[str],
  1119. ):
  1120. self.connection_pool = connection_pool
  1121. self.connection = None
  1122. self.response_callbacks = response_callbacks
  1123. self.is_transaction = transaction
  1124. self.shard_hint = shard_hint
  1125. self.watching = False
  1126. self.command_stack: CommandStackT = []
  1127. self.scripts: Set[Script] = set()
  1128. self.explicit_transaction = False
  1129. async def __aenter__(self: _RedisT) -> _RedisT:
  1130. return self
  1131. async def __aexit__(self, exc_type, exc_value, traceback):
  1132. await self.reset()
  1133. def __await__(self):
  1134. return self._async_self().__await__()
  1135. _DEL_MESSAGE = "Unclosed Pipeline client"
  1136. def __len__(self):
  1137. return len(self.command_stack)
  1138. def __bool__(self):
  1139. """Pipeline instances should always evaluate to True"""
  1140. return True
  1141. async def _async_self(self):
  1142. return self
  1143. async def reset(self):
  1144. self.command_stack = []
  1145. self.scripts = set()
  1146. # make sure to reset the connection state in the event that we were
  1147. # watching something
  1148. if self.watching and self.connection:
  1149. try:
  1150. # call this manually since our unwatch or
  1151. # immediate_execute_command methods can call reset()
  1152. await self.connection.send_command("UNWATCH")
  1153. await self.connection.read_response()
  1154. except ConnectionError:
  1155. # disconnect will also remove any previous WATCHes
  1156. if self.connection:
  1157. await self.connection.disconnect()
  1158. # clean up the other instance attributes
  1159. self.watching = False
  1160. self.explicit_transaction = False
  1161. # we can safely return the connection to the pool here since we're
  1162. # sure we're no longer WATCHing anything
  1163. if self.connection:
  1164. await self.connection_pool.release(self.connection)
  1165. self.connection = None
  1166. async def aclose(self) -> None:
  1167. """Alias for reset(), a standard method name for cleanup"""
  1168. await self.reset()
  1169. def multi(self):
  1170. """
  1171. Start a transactional block of the pipeline after WATCH commands
  1172. are issued. End the transactional block with `execute`.
  1173. """
  1174. if self.explicit_transaction:
  1175. raise RedisError("Cannot issue nested calls to MULTI")
  1176. if self.command_stack:
  1177. raise RedisError(
  1178. "Commands without an initial WATCH have already been issued"
  1179. )
  1180. self.explicit_transaction = True
  1181. def execute_command(
  1182. self, *args, **kwargs
  1183. ) -> Union["Pipeline", Awaitable["Pipeline"]]:
  1184. if (self.watching or args[0] == "WATCH") and not self.explicit_transaction:
  1185. return self.immediate_execute_command(*args, **kwargs)
  1186. return self.pipeline_execute_command(*args, **kwargs)
  1187. async def _disconnect_reset_raise_on_watching(
  1188. self,
  1189. conn: Connection,
  1190. error: Exception,
  1191. ):
  1192. """
  1193. Close the connection reset watching state and
  1194. raise an exception if we were watching.
  1195. The supported exceptions are already checked in the
  1196. retry object so we don't need to do it here.
  1197. After we disconnect the connection, it will try to reconnect and
  1198. do a health check as part of the send_command logic(on connection level).
  1199. """
  1200. await conn.disconnect()
  1201. # if we were already watching a variable, the watch is no longer
  1202. # valid since this connection has died. raise a WatchError, which
  1203. # indicates the user should retry this transaction.
  1204. if self.watching:
  1205. await self.reset()
  1206. raise WatchError(
  1207. f"A {type(error).__name__} occurred while watching one or more keys"
  1208. )
  1209. async def immediate_execute_command(self, *args, **options):
  1210. """
  1211. Execute a command immediately, but don't auto-retry on the supported
  1212. errors for retry if we're already WATCHing a variable.
  1213. Used when issuing WATCH or subsequent commands retrieving their values but before
  1214. MULTI is called.
  1215. """
  1216. command_name = args[0]
  1217. conn = self.connection
  1218. # if this is the first call, we need a connection
  1219. if not conn:
  1220. conn = await self.connection_pool.get_connection()
  1221. self.connection = conn
  1222. return await conn.retry.call_with_retry(
  1223. lambda: self._send_command_parse_response(
  1224. conn, command_name, *args, **options
  1225. ),
  1226. lambda error: self._disconnect_reset_raise_on_watching(conn, error),
  1227. )
  1228. def pipeline_execute_command(self, *args, **options):
  1229. """
  1230. Stage a command to be executed when execute() is next called
  1231. Returns the current Pipeline object back so commands can be
  1232. chained together, such as:
  1233. pipe = pipe.set('foo', 'bar').incr('baz').decr('bang')
  1234. At some other point, you can then run: pipe.execute(),
  1235. which will execute all commands queued in the pipe.
  1236. """
  1237. self.command_stack.append((args, options))
  1238. return self
  1239. async def _execute_transaction( # noqa: C901
  1240. self, connection: Connection, commands: CommandStackT, raise_on_error
  1241. ):
  1242. pre: CommandT = (("MULTI",), {})
  1243. post: CommandT = (("EXEC",), {})
  1244. cmds = (pre, *commands, post)
  1245. all_cmds = connection.pack_commands(
  1246. args for args, options in cmds if EMPTY_RESPONSE not in options
  1247. )
  1248. await connection.send_packed_command(all_cmds)
  1249. errors = []
  1250. # parse off the response for MULTI
  1251. # NOTE: we need to handle ResponseErrors here and continue
  1252. # so that we read all the additional command messages from
  1253. # the socket
  1254. try:
  1255. await self.parse_response(connection, "_")
  1256. except ResponseError as err:
  1257. errors.append((0, err))
  1258. # and all the other commands
  1259. for i, command in enumerate(commands):
  1260. if EMPTY_RESPONSE in command[1]:
  1261. errors.append((i, command[1][EMPTY_RESPONSE]))
  1262. else:
  1263. try:
  1264. await self.parse_response(connection, "_")
  1265. except ResponseError as err:
  1266. self.annotate_exception(err, i + 1, command[0])
  1267. errors.append((i, err))
  1268. # parse the EXEC.
  1269. try:
  1270. response = await self.parse_response(connection, "_")
  1271. except ExecAbortError as err:
  1272. if errors:
  1273. raise errors[0][1] from err
  1274. raise
  1275. # EXEC clears any watched keys
  1276. self.watching = False
  1277. if response is None:
  1278. raise WatchError("Watched variable changed.") from None
  1279. # put any parse errors into the response
  1280. for i, e in errors:
  1281. response.insert(i, e)
  1282. if len(response) != len(commands):
  1283. if self.connection:
  1284. await self.connection.disconnect()
  1285. raise ResponseError(
  1286. "Wrong number of response items from pipeline execution"
  1287. ) from None
  1288. # find any errors in the response and raise if necessary
  1289. if raise_on_error:
  1290. self.raise_first_error(commands, response)
  1291. # We have to run response callbacks manually
  1292. data = []
  1293. for r, cmd in zip(response, commands):
  1294. if not isinstance(r, Exception):
  1295. args, options = cmd
  1296. command_name = args[0]
  1297. # Remove keys entry, it needs only for cache.
  1298. options.pop("keys", None)
  1299. if command_name in self.response_callbacks:
  1300. r = self.response_callbacks[command_name](r, **options)
  1301. if inspect.isawaitable(r):
  1302. r = await r
  1303. data.append(r)
  1304. return data
  1305. async def _execute_pipeline(
  1306. self, connection: Connection, commands: CommandStackT, raise_on_error: bool
  1307. ):
  1308. # build up all commands into a single request to increase network perf
  1309. all_cmds = connection.pack_commands([args for args, _ in commands])
  1310. await connection.send_packed_command(all_cmds)
  1311. response = []
  1312. for args, options in commands:
  1313. try:
  1314. response.append(
  1315. await self.parse_response(connection, args[0], **options)
  1316. )
  1317. except ResponseError as e:
  1318. response.append(e)
  1319. if raise_on_error:
  1320. self.raise_first_error(commands, response)
  1321. return response
  1322. def raise_first_error(self, commands: CommandStackT, response: Iterable[Any]):
  1323. for i, r in enumerate(response):
  1324. if isinstance(r, ResponseError):
  1325. self.annotate_exception(r, i + 1, commands[i][0])
  1326. raise r
  1327. def annotate_exception(
  1328. self, exception: Exception, number: int, command: Iterable[object]
  1329. ) -> None:
  1330. cmd = " ".join(map(safe_str, command))
  1331. msg = (
  1332. f"Command # {number} ({truncate_text(cmd)}) "
  1333. "of pipeline caused error: {exception.args}"
  1334. )
  1335. exception.args = (msg,) + exception.args[1:]
  1336. async def parse_response(
  1337. self, connection: Connection, command_name: Union[str, bytes], **options
  1338. ):
  1339. result = await super().parse_response(connection, command_name, **options)
  1340. if command_name in self.UNWATCH_COMMANDS:
  1341. self.watching = False
  1342. elif command_name == "WATCH":
  1343. self.watching = True
  1344. return result
  1345. async def load_scripts(self):
  1346. # make sure all scripts that are about to be run on this pipeline exist
  1347. scripts = list(self.scripts)
  1348. immediate = self.immediate_execute_command
  1349. shas = [s.sha for s in scripts]
  1350. # we can't use the normal script_* methods because they would just
  1351. # get buffered in the pipeline.
  1352. exists = await immediate("SCRIPT EXISTS", *shas)
  1353. if not all(exists):
  1354. for s, exist in zip(scripts, exists):
  1355. if not exist:
  1356. s.sha = await immediate("SCRIPT LOAD", s.script)
  1357. async def _disconnect_raise_on_watching(self, conn: Connection, error: Exception):
  1358. """
  1359. Close the connection, raise an exception if we were watching.
  1360. The supported exceptions are already checked in the
  1361. retry object so we don't need to do it here.
  1362. After we disconnect the connection, it will try to reconnect and
  1363. do a health check as part of the send_command logic(on connection level).
  1364. """
  1365. await conn.disconnect()
  1366. # if we were watching a variable, the watch is no longer valid
  1367. # since this connection has died. raise a WatchError, which
  1368. # indicates the user should retry this transaction.
  1369. if self.watching:
  1370. raise WatchError(
  1371. f"A {type(error).__name__} occurred while watching one or more keys"
  1372. )
  1373. async def execute(self, raise_on_error: bool = True) -> List[Any]:
  1374. """Execute all the commands in the current pipeline"""
  1375. stack = self.command_stack
  1376. if not stack and not self.watching:
  1377. return []
  1378. if self.scripts:
  1379. await self.load_scripts()
  1380. if self.is_transaction or self.explicit_transaction:
  1381. execute = self._execute_transaction
  1382. else:
  1383. execute = self._execute_pipeline
  1384. conn = self.connection
  1385. if not conn:
  1386. conn = await self.connection_pool.get_connection()
  1387. # assign to self.connection so reset() releases the connection
  1388. # back to the pool after we're done
  1389. self.connection = conn
  1390. conn = cast(Connection, conn)
  1391. try:
  1392. return await conn.retry.call_with_retry(
  1393. lambda: execute(conn, stack, raise_on_error),
  1394. lambda error: self._disconnect_raise_on_watching(conn, error),
  1395. )
  1396. finally:
  1397. await self.reset()
  1398. async def discard(self):
  1399. """Flushes all previously queued commands
  1400. See: https://redis.io/commands/DISCARD
  1401. """
  1402. await self.execute_command("DISCARD")
  1403. async def watch(self, *names: KeyT):
  1404. """Watches the values at keys ``names``"""
  1405. if self.explicit_transaction:
  1406. raise RedisError("Cannot issue a WATCH after a MULTI")
  1407. return await self.execute_command("WATCH", *names)
  1408. async def unwatch(self):
  1409. """Unwatches all previously specified keys"""
  1410. return self.watching and await self.execute_command("UNWATCH") or True