rethinkdb.py 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155
  1. from __future__ import absolute_import
  2. from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
  3. from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
  4. from apscheduler.job import Job
  5. try:
  6. import cPickle as pickle
  7. except ImportError: # pragma: nocover
  8. import pickle
  9. try:
  10. from rethinkdb import RethinkDB
  11. except ImportError: # pragma: nocover
  12. raise ImportError('RethinkDBJobStore requires rethinkdb installed')
  13. class RethinkDBJobStore(BaseJobStore):
  14. """
  15. Stores jobs in a RethinkDB database. Any leftover keyword arguments are directly passed to
  16. rethinkdb's `RethinkdbClient <http://www.rethinkdb.com/api/#connect>`_.
  17. Plugin alias: ``rethinkdb``
  18. :param str database: database to store jobs in
  19. :param str collection: collection to store jobs in
  20. :param client: a :class:`rethinkdb.net.Connection` instance to use instead of providing
  21. connection arguments
  22. :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
  23. highest available
  24. """
  25. def __init__(self, database='apscheduler', table='jobs', client=None,
  26. pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
  27. super(RethinkDBJobStore, self).__init__()
  28. if not database:
  29. raise ValueError('The "database" parameter must not be empty')
  30. if not table:
  31. raise ValueError('The "table" parameter must not be empty')
  32. self.database = database
  33. self.table_name = table
  34. self.table = None
  35. self.client = client
  36. self.pickle_protocol = pickle_protocol
  37. self.connect_args = connect_args
  38. self.r = RethinkDB()
  39. self.conn = None
  40. def start(self, scheduler, alias):
  41. super(RethinkDBJobStore, self).start(scheduler, alias)
  42. if self.client:
  43. self.conn = maybe_ref(self.client)
  44. else:
  45. self.conn = self.r.connect(db=self.database, **self.connect_args)
  46. if self.database not in self.r.db_list().run(self.conn):
  47. self.r.db_create(self.database).run(self.conn)
  48. if self.table_name not in self.r.table_list().run(self.conn):
  49. self.r.table_create(self.table_name).run(self.conn)
  50. if 'next_run_time' not in self.r.table(self.table_name).index_list().run(self.conn):
  51. self.r.table(self.table_name).index_create('next_run_time').run(self.conn)
  52. self.table = self.r.db(self.database).table(self.table_name)
  53. def lookup_job(self, job_id):
  54. results = list(self.table.get_all(job_id).pluck('job_state').run(self.conn))
  55. return self._reconstitute_job(results[0]['job_state']) if results else None
  56. def get_due_jobs(self, now):
  57. return self._get_jobs(self.r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
  58. def get_next_run_time(self):
  59. results = list(
  60. self.table
  61. .filter(self.r.row['next_run_time'] != None) # noqa
  62. .order_by(self.r.asc('next_run_time'))
  63. .map(lambda x: x['next_run_time'])
  64. .limit(1)
  65. .run(self.conn)
  66. )
  67. return utc_timestamp_to_datetime(results[0]) if results else None
  68. def get_all_jobs(self):
  69. jobs = self._get_jobs()
  70. self._fix_paused_jobs_sorting(jobs)
  71. return jobs
  72. def add_job(self, job):
  73. job_dict = {
  74. 'id': job.id,
  75. 'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
  76. 'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
  77. }
  78. results = self.table.insert(job_dict).run(self.conn)
  79. if results['errors'] > 0:
  80. raise ConflictingIdError(job.id)
  81. def update_job(self, job):
  82. changes = {
  83. 'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
  84. 'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
  85. }
  86. results = self.table.get_all(job.id).update(changes).run(self.conn)
  87. skipped = False in map(lambda x: results[x] == 0, results.keys())
  88. if results['skipped'] > 0 or results['errors'] > 0 or not skipped:
  89. raise JobLookupError(job.id)
  90. def remove_job(self, job_id):
  91. results = self.table.get_all(job_id).delete().run(self.conn)
  92. if results['deleted'] + results['skipped'] != 1:
  93. raise JobLookupError(job_id)
  94. def remove_all_jobs(self):
  95. self.table.delete().run(self.conn)
  96. def shutdown(self):
  97. self.conn.close()
  98. def _reconstitute_job(self, job_state):
  99. job_state = pickle.loads(job_state)
  100. job = Job.__new__(Job)
  101. job.__setstate__(job_state)
  102. job._scheduler = self._scheduler
  103. job._jobstore_alias = self._alias
  104. return job
  105. def _get_jobs(self, predicate=None):
  106. jobs = []
  107. failed_job_ids = []
  108. query = (self.table.filter(self.r.row['next_run_time'] != None).filter(predicate) # noqa
  109. if predicate else self.table)
  110. query = query.order_by('next_run_time', 'id').pluck('id', 'job_state')
  111. for document in query.run(self.conn):
  112. try:
  113. jobs.append(self._reconstitute_job(document['job_state']))
  114. except Exception:
  115. self._logger.exception('Unable to restore job "%s" -- removing it', document['id'])
  116. failed_job_ids.append(document['id'])
  117. # Remove all the jobs we failed to restore
  118. if failed_job_ids:
  119. self.r.expr(failed_job_ids).for_each(
  120. lambda job_id: self.table.get_all(job_id).delete()).run(self.conn)
  121. return jobs
  122. def __repr__(self):
  123. connection = self.conn
  124. return '<%s (connection=%s)>' % (self.__class__.__name__, connection)