retry.py 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249
  1. import random
  2. from typing import List, Any, Dict
  3. MAX_DELAY_TIME = 120 * 1000
  4. MIN_DELAY_TIME = 100
  5. _SNAKE_TO_CAMEL = {
  6. 'max_attempts': 'maxAttempts',
  7. 'error_code': 'errorCode',
  8. 'max_delay': 'maxDelay',
  9. 'retry_condition': 'retryCondition',
  10. 'no_retry_condition': 'noRetryCondition',
  11. }
  12. def _normalize_option_keys(data: Dict[str, Any]) -> Dict[str, Any]:
  13. result: Dict[str, Any] = {}
  14. for key, value in data.items():
  15. result[_SNAKE_TO_CAMEL.get(key, key)] = value
  16. return result
  17. def _merge_option_data(option: Dict[str, Any] = None, **kwargs) -> Dict[str, Any]:
  18. data: Dict[str, Any] = {}
  19. if option is not None:
  20. if not isinstance(option, dict):
  21. raise TypeError('option must be a dict')
  22. data.update(option)
  23. if kwargs:
  24. data.update(_normalize_option_keys(kwargs))
  25. return data
  26. class BackoffPolicy:
  27. def __init__(self, option: Dict[str, Any]):
  28. self.policy = option.get("policy")
  29. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  30. raise NotImplementedError('un-implemented')
  31. @staticmethod
  32. def new_backoff_policy(option: Dict[str, Any]) -> 'BackoffPolicy':
  33. policy_map = {
  34. 'Fixed': FixedBackoffPolicy,
  35. 'Random': RandomBackoffPolicy,
  36. 'Exponential': ExponentialBackoffPolicy,
  37. 'EqualJitter': EqualJitterBackoffPolicy,
  38. 'ExponentialWithEqualJitter': EqualJitterBackoffPolicy,
  39. 'FullJitter': FullJitterBackoffPolicy,
  40. 'ExponentialWithFullJitter': FullJitterBackoffPolicy,
  41. }
  42. policy_class = policy_map.get(option.get('policy'))
  43. if policy_class:
  44. return policy_class(option)
  45. raise ValueError(f"Unknown policy: {option.get('policy')}")
  46. class FixedBackoffPolicy(BackoffPolicy):
  47. def __init__(self, option: Dict[str, Any]):
  48. super().__init__(option)
  49. self.period = option.get('period')
  50. def to_map(self):
  51. return {
  52. 'policy': self.policy,
  53. 'period': self.period,
  54. }
  55. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  56. return self.period
  57. class RandomBackoffPolicy(BackoffPolicy):
  58. def __init__(self, option: Dict[str, Any]):
  59. super().__init__(option)
  60. self.period = option.get('period')
  61. self.cap = option.get('cap', 20 * 1000)
  62. def to_map(self):
  63. return {
  64. 'policy': self.policy,
  65. 'period': self.period,
  66. 'cap': self.cap,
  67. }
  68. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  69. random_time = random.randint(0, ctx.retries_attempted * self.period)
  70. return min(random_time, self.cap)
  71. class ExponentialBackoffPolicy(BackoffPolicy):
  72. def __init__(self, option: Dict[str, Any]):
  73. super().__init__(option)
  74. self.period = option.get('period')
  75. self.cap = option.get('cap', 3 * 24 * 60 * 60 * 1000)
  76. def to_map(self):
  77. return {
  78. 'policy': self.policy,
  79. 'period': self.period,
  80. 'cap': self.cap,
  81. }
  82. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  83. # period * 2^retries (align with C# / standard exponential backoff)
  84. random_time = min(self.period * (2 ** ctx.retries_attempted), self.cap)
  85. return random_time
  86. class EqualJitterBackoffPolicy(BackoffPolicy):
  87. def __init__(self, option: Dict[str, Any]):
  88. super().__init__(option)
  89. self.period = option.get('period')
  90. self.cap = option.get('cap', 3 * 24 * 60 * 60 * 1000)
  91. def to_map(self):
  92. return {
  93. 'policy': self.policy,
  94. 'period': self.period,
  95. 'cap': self.cap,
  96. }
  97. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  98. ceil = min(self.cap, self.period * (2 ** ctx.retries_attempted))
  99. return ceil // 2 + random.randint(0, ceil // 2)
  100. class FullJitterBackoffPolicy(BackoffPolicy):
  101. def __init__(self, option: Dict[str, Any]):
  102. super().__init__(option)
  103. self.period = option.get('period')
  104. self.cap = option.get('cap', 3 * 24 * 60 * 60 * 1000)
  105. def to_map(self):
  106. return {
  107. 'policy': self.policy,
  108. 'period': self.period,
  109. 'cap': self.cap,
  110. }
  111. def get_delay_time(self, ctx: 'RetryPolicyContext') -> int:
  112. ceil = min(self.cap, self.period * (2 ** ctx.retries_attempted))
  113. return random.randint(0, ceil)
  114. class RetryCondition:
  115. def __init__(self, condition: Dict[str, Any] = None, **kwargs):
  116. data = _merge_option_data(condition, **kwargs)
  117. self.max_attempts = data.get('maxAttempts', None)
  118. self.backoff = self._ensure_backoff_policy(data.get('backoff', None))
  119. self.exception = data.get('exception', [])
  120. self.error_code = data.get('errorCode', [])
  121. self.max_delay = data.get('maxDelay', None)
  122. def _ensure_backoff_policy(self, backoff):
  123. if isinstance(backoff, dict):
  124. return BackoffPolicy.new_backoff_policy(backoff)
  125. elif isinstance(backoff, BackoffPolicy):
  126. return backoff
  127. def to_map(self):
  128. result = dict()
  129. if self.max_attempts:
  130. result['maxAttempts'] = self.max_attempts
  131. if self.backoff:
  132. result['backoff'] = self.backoff.to_map()
  133. if self.exception:
  134. result['exception'] = self.exception
  135. if self.error_code:
  136. result['errorCode'] = self.error_code
  137. if self.max_delay:
  138. result['maxDelay'] = self.max_delay
  139. return result
  140. @staticmethod
  141. def from_map(data: Dict[str, Any]) -> 'RetryCondition':
  142. return RetryCondition({
  143. 'maxAttempts': data.get('maxAttempts'),
  144. 'backoff': data.get('backoff'),
  145. 'exception': data.get('exception', []),
  146. 'errorCode': data.get('errorCode', []),
  147. 'maxDelay': data.get('maxDelay')
  148. })
  149. class RetryOptions:
  150. def __init__(self, options: Dict[str, Any] = None, **kwargs):
  151. data = _merge_option_data(options, **kwargs)
  152. self.retryable = data.get('retryable', True)
  153. self.max_attempts = data.get('maxAttempts', None)
  154. self.retry_condition = [self._ensure_retry_condition(cond) for cond in data.get('retryCondition', [])]
  155. self.no_retry_condition = [self._ensure_retry_condition(cond) for cond in data.get('noRetryCondition', [])]
  156. def _ensure_retry_condition(self, condition):
  157. if isinstance(condition, dict):
  158. return RetryCondition(condition)
  159. elif isinstance(condition, RetryCondition):
  160. return condition
  161. else:
  162. raise ValueError("Condition must be either a dictionary or a RetryCondition instance")
  163. def validate(self) -> bool:
  164. if not isinstance(self.retryable, bool):
  165. raise ValueError("retryable must be a boolean.")
  166. if not isinstance(self.retry_condition, list) or not all(isinstance(cond, RetryCondition) for cond in self.retry_condition):
  167. raise ValueError("retryCondition must be a list of RetryCondition.")
  168. if not isinstance(self.no_retry_condition, list) or not all(isinstance(cond, RetryCondition) for cond in self.no_retry_condition):
  169. raise ValueError("noRetryCondition must be a list of RetryCondition.")
  170. return True
  171. def to_map(self):
  172. result = dict()
  173. if self.retryable:
  174. result['retryable'] = self.retryable
  175. if self.retry_condition:
  176. result['retryCondition'] = [cond.to_map() for cond in self.retry_condition]
  177. if self.no_retry_condition:
  178. result['noRetryCondition'] = [cond.to_map() for cond in self.no_retry_condition]
  179. return result
  180. @staticmethod
  181. def from_map(data: Dict[str, Any]) -> 'RetryOptions':
  182. options = {
  183. 'retryable': data.get('retryable', True),
  184. 'retryCondition': [cond for cond in data.get('retryCondition', [])],
  185. 'noRetryCondition': [cond for cond in data.get('noRetryCondition', [])]
  186. }
  187. return RetryOptions(options)
  188. class RetryPolicyContext:
  189. def __init__(self, retries_attempted = None, http_request = None, http_response = None, exception = None):
  190. self.retries_attempted = retries_attempted
  191. self.http_request = http_request
  192. self.http_response = http_response
  193. self.exception = exception
  194. def get_backoff_delay(options: RetryOptions, ctx: RetryPolicyContext) -> int:
  195. ex = ctx.exception
  196. for condition in options.retry_condition:
  197. ex_name = getattr(ex, 'name', None)
  198. ex_code = getattr(ex, 'code', None)
  199. if ex and (ex_name in condition.exception or ex_code in condition.error_code):
  200. max_delay = condition.max_delay or MAX_DELAY_TIME
  201. retry_after = getattr(ex, 'retry_after', None)
  202. if retry_after is None:
  203. retry_after = getattr(ex, 'retryAfter', None)
  204. if retry_after is not None:
  205. return min(retry_after, max_delay)
  206. if not condition.backoff:
  207. return MIN_DELAY_TIME
  208. return min(condition.backoff.get_delay_time(ctx), max_delay)
  209. return MIN_DELAY_TIME