retry.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. import random
  2. from typing import List, Any, Dict
  3. MAX_DELAY_TIME = 120 * 1000
  4. MIN_DELAY_TIME = 100
  5. _SNAKE_TO_CAMEL = {
  6. 'max_attempts': 'maxAttempts',
  7. 'error_code': 'errorCode',
  8. 'max_delay': 'maxDelay',
  9. 'retry_condition': 'retryCondition',
  10. 'no_retry_condition': 'noRetryCondition',
  11. }
  12. def _normalize_option_keys(data: Dict[str, Any]) -> Dict[str, Any]:
  13. result: Dict[str, Any] = {}
  14. for key, value in data.items():
  15. result[_SNAKE_TO_CAMEL.get(key, key)] = value
  16. return result
  17. def _merge_option_data(option: Dict[str, Any] = None, **kwargs) -> Dict[str, Any]:
  18. data: Dict[str, Any] = {}
  19. if option is not None:
  20. if not isinstance(option, dict):
  21. raise TypeError('option must be a dict')
  22. data.update(option)
  23. if kwargs:
  24. data.update(_normalize_option_keys(kwargs))
  25. return data
  26. class BackoffPolicy:
  27. def __init__(self, option: Dict[str, Any]):
  28. self.policy = option.get("policy")
  29. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  30. raise NotImplementedError('un-implemented')
  31. @staticmethod
  32. def new_backoff_policy(option: Dict[str, Any]) -> 'BackoffPolicy':
  33. policy_map = {
  34. 'Fixed': FixedBackoffPolicy,
  35. 'Random': RandomBackoffPolicy,
  36. 'Exponential': ExponentialBackoffPolicy,
  37. 'EqualJitter': EqualJitterBackoffPolicy,
  38. 'ExponentialWithEqualJitter': EqualJitterBackoffPolicy,
  39. 'FullJitter': FullJitterBackoffPolicy,
  40. 'ExponentialWithFullJitter': FullJitterBackoffPolicy,
  41. }
  42. policy_class = policy_map.get(option.get('policy'))
  43. if policy_class:
  44. return policy_class(option)
  45. raise ValueError(f"Unknown policy: {option.get('policy')}")
  46. class FixedBackoffPolicy(BackoffPolicy):
  47. def __init__(self, option: Dict[str, Any]):
  48. super().__init__(option)
  49. self.period = option.get('period')
  50. def to_map(self):
  51. return {
  52. 'policy': self.policy,
  53. 'period': self.period,
  54. }
  55. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  56. return self.period
  57. class RandomBackoffPolicy(BackoffPolicy):
  58. def __init__(self, option: Dict[str, Any]):
  59. super().__init__(option)
  60. self.period = option.get('period')
  61. self.cap = option.get('cap', 20 * 1000)
  62. def to_map(self):
  63. return {
  64. 'policy': self.policy,
  65. 'period': self.period,
  66. 'cap': self.cap,
  67. }
  68. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  69. random_time = random.randint(0, ctx.retries_attempted * self.period)
  70. return min(random_time, self.cap)
  71. class ExponentialBackoffPolicy(BackoffPolicy):
  72. def __init__(self, option: Dict[str, Any]):
  73. super().__init__(option)
  74. self.period = option.get('period')
  75. self.cap = option.get('cap', 3 * 24 * 60 * 60 * 1000)
  76. def to_map(self):
  77. return {
  78. 'policy': self.policy,
  79. 'period': self.period,
  80. 'cap': self.cap,
  81. }
  82. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  83. random_time = min(2 ** (ctx.retries_attempted * self.period), self.cap)
  84. return random_time
  85. class EqualJitterBackoffPolicy(BackoffPolicy):
  86. def __init__(self, option: Dict[str, Any]):
  87. super().__init__(option)
  88. self.period = option.get('period')
  89. self.cap = option.get('cap', 3 * 24 * 60 * 60 * 1000)
  90. def to_map(self):
  91. return {
  92. 'policy': self.policy,
  93. 'period': self.period,
  94. 'cap': self.cap,
  95. }
  96. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  97. ceil = min(self.cap, 2 ** (ctx.retries_attempted * self.period))
  98. return ceil // 2 + random.randint(0, ceil // 2)
  99. class FullJitterBackoffPolicy(BackoffPolicy):
  100. def __init__(self, option: Dict[str, Any]):
  101. super().__init__(option)
  102. self.period = option.get('period')
  103. self.cap = option.get('cap', 3 * 24 * 60 * 60 * 1000)
  104. def to_map(self):
  105. return {
  106. 'policy': self.policy,
  107. 'period': self.period,
  108. 'cap': self.cap,
  109. }
  110. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  111. ceil = min(self.cap, 2 ** (ctx.retries_attempted * self.period))
  112. return random.randint(0, ceil)
  113. class RetryCondition:
  114. def __init__(self, condition: Dict[str, Any] = None, **kwargs):
  115. data = _merge_option_data(condition, **kwargs)
  116. self.max_attempts = data.get('maxAttempts', None)
  117. self.backoff = self._ensure_backoff_policy(data.get('backoff', None))
  118. self.exception = data.get('exception', [])
  119. self.error_code = data.get('errorCode', [])
  120. self.max_delay = data.get('maxDelay', None)
  121. def _ensure_backoff_policy(self, backoff):
  122. if isinstance(backoff, dict):
  123. return BackoffPolicy.new_backoff_policy(backoff)
  124. elif isinstance(backoff, BackoffPolicy):
  125. return backoff
  126. def to_map(self):
  127. result = dict()
  128. if self.max_attempts:
  129. result['maxAttempts'] = self.max_attempts
  130. if self.backoff:
  131. result['backoff'] = self.backoff.to_map()
  132. if self.exception:
  133. result['exception'] = self.exception
  134. if self.error_code:
  135. result['errorCode'] = self.error_code
  136. if self.max_delay:
  137. result['maxDelay'] = self.max_delay
  138. return result
  139. @staticmethod
  140. def from_map(data: Dict[str, Any]) -> 'RetryCondition':
  141. return RetryCondition({
  142. 'maxAttempts': data.get('maxAttempts'),
  143. 'backoff': data.get('backoff'),
  144. 'exception': data.get('exception', []),
  145. 'errorCode': data.get('errorCode', []),
  146. 'maxDelay': data.get('maxDelay')
  147. })
  148. class RetryOptions:
  149. def __init__(self, options: Dict[str, Any] = None, **kwargs):
  150. data = _merge_option_data(options, **kwargs)
  151. self.retryable = data.get('retryable', True)
  152. self.max_attempts = data.get('maxAttempts', None)
  153. self.retry_condition = [self._ensure_retry_condition(cond) for cond in data.get('retryCondition', [])]
  154. self.no_retry_condition = [self._ensure_retry_condition(cond) for cond in data.get('noRetryCondition', [])]
  155. def _ensure_retry_condition(self, condition):
  156. if isinstance(condition, dict):
  157. return RetryCondition(condition)
  158. elif isinstance(condition, RetryCondition):
  159. return condition
  160. else:
  161. raise ValueError("Condition must be either a dictionary or a RetryCondition instance")
  162. def validate(self) -> bool:
  163. if not isinstance(self.retryable, bool):
  164. raise ValueError("retryable must be a boolean.")
  165. if not isinstance(self.retry_condition, list) or not all(isinstance(cond, RetryCondition) for cond in self.retry_condition):
  166. raise ValueError("retryCondition must be a list of RetryCondition.")
  167. if not isinstance(self.no_retry_condition, list) or not all(isinstance(cond, RetryCondition) for cond in self.no_retry_condition):
  168. raise ValueError("noRetryCondition must be a list of RetryCondition.")
  169. return True
  170. def to_map(self):
  171. result = dict()
  172. if self.retryable:
  173. result['retryable'] = self.retryable
  174. if self.retry_condition:
  175. result['retryCondition'] = [cond.to_map() for cond in self.retry_condition]
  176. if self.no_retry_condition:
  177. result['noRetryCondition'] = [cond.to_map() for cond in self.no_retry_condition]
  178. return result
  179. @staticmethod
  180. def from_map(data: Dict[str, Any]) -> 'RetryOptions':
  181. options = {
  182. 'retryable': data.get('retryable', True),
  183. 'retryCondition': [cond for cond in data.get('retryCondition', [])],
  184. 'noRetryCondition': [cond for cond in data.get('noRetryCondition', [])]
  185. }
  186. return RetryOptions(options)
  187. class RetryPolicyContext:
  188. def __init__(self, retries_attempted = None, http_request = None, http_response = None, exception = None):
  189. self.retries_attempted = retries_attempted
  190. self.http_request = http_request
  191. self.http_response = http_response
  192. self.exception = exception
  193. def get_backoff_delay(options: RetryOptions, ctx: RetryPolicyContext) -> int:
  194. ex = ctx.exception
  195. for condition in options.retry_condition:
  196. ex_name = getattr(ex, 'name', None)
  197. ex_code = getattr(ex, 'code', None)
  198. if ex and (ex_name in condition.exception or ex_code in condition.error_code):
  199. max_delay = condition.max_delay or MAX_DELAY_TIME
  200. retry_after = getattr(ex, 'retry_after', None)
  201. if retry_after is None:
  202. retry_after = getattr(ex, 'retryAfter', None)
  203. if retry_after is not None:
  204. return min(retry_after, max_delay)
  205. if not condition.backoff:
  206. return MIN_DELAY_TIME
  207. return min(condition.backoff.get_delay_time(ctx), max_delay)
  208. return MIN_DELAY_TIME