Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
from huey import RedisHuey
huey = RedisHuey('simple.test', blocking=True)
def start(self):
if self.huey_settings is None:
self._no_config()
elif isinstance(self.huey_settings, RedisHuey):
self._huey_instance_config()
elif isinstance(self.huey_settings, dict):
if SingleConfReader.is_legacy(self.huey_settings):
self._single_config()
elif MultiConfReader.is_multi_config(self.huey_settings):
self._multi_config()
else:
raise ConfigurationError('HUEY settings dictionary invalid.')
else:
raise ConfigurationError('Configuration doesnt match guidelines.')
import os
task_runner = None
try:
import redis
from huey import RedisHuey
task_runner = RedisHuey(
"pntl_task", host=os.getenv("REDISH_HOST", "redis://127.0.0.1:6379/1")
)
except ImportError:
from huey.contrib.sqlitedb import SqliteHuey
task_runner = SqliteHuey("pntl_task.db")
from huey import RedisHuey
huey = RedisHuey()
from app.util.task.tasks import example
if __name__ == '__main__':
print('Usage:')
print(' huey_consumer.py run_huey.huey')
pass
from huey import RedisHuey, crontab
from portality.core import app
main_queue = RedisHuey('doaj_main_queue', host=app.config['HUEY_REDIS_HOST'], port=app.config['HUEY_REDIS_PORT'], always_eager=app.config.get("HUEY_EAGER", False))
long_running = RedisHuey('doaj_long_running', host=app.config['HUEY_REDIS_HOST'], port=app.config['HUEY_REDIS_PORT'], always_eager=app.config.get("HUEY_EAGER", False))
def schedule(action):
cfg = app.config.get("HUEY_SCHEDULE", {})
action_cfg = cfg.get(action)
if action_cfg is None:
raise RuntimeError("No configuration for scheduled action '{x}'. Define this in HUEY_SCHEDULE first then try again.".format(x=action))
return crontab(**action_cfg)
def configure(action):
cfg = app.config.get("HUEY_TASKS", {})
action_cfg = cfg.get(action)
if action_cfg is None:
raise RuntimeError("No task configuration for action '{x}'. Define this in HUEY_TASKS first then try again.".format(x=action))
def _no_config(self):
try:
from huey import RedisHuey
except ImportError:
config_error('Error: Huey could not import the redis backend. '
'Install `redis-py`.')
else:
self.huey = RedisHuey(default_queue_name())
self.hueys = {default_queue_name(): self.huey}
from huey import RedisHuey, crontab
from portality.core import app
main_queue = RedisHuey('doaj_main_queue', host=app.config['HUEY_REDIS_HOST'], port=app.config['HUEY_REDIS_PORT'], always_eager=app.config.get("HUEY_EAGER", False))
long_running = RedisHuey('doaj_long_running', host=app.config['HUEY_REDIS_HOST'], port=app.config['HUEY_REDIS_PORT'], always_eager=app.config.get("HUEY_EAGER", False))
def schedule(action):
cfg = app.config.get("HUEY_SCHEDULE", {})
action_cfg = cfg.get(action)
if action_cfg is None:
raise RuntimeError("No configuration for scheduled action '{x}'. Define this in HUEY_SCHEDULE first then try again.".format(x=action))
return crontab(**action_cfg)
def configure(action):
cfg = app.config.get("HUEY_TASKS", {})
action_cfg = cfg.get(action)
#coding=UTF-8
import importlib
import time
import redis
from huey import RedisHuey
from config import *
from sutils import filter
from sutils import queues
task = importlib.import_module(TASK_MODULE)
pool = redis.ConnectionPool(host=RTASK_REDIS_HOST, port=RTASK_REDIS_POST, db=RTASK_REDIS_DB, password=RTASK_REDIS_PWD)
huey = RedisHuey(TASK_NAME, connection_pool=pool, result_store=False)
#获取去重对象
def get_filter(filter_type='set', redis_type='single', host='127.0.0.1', port=6379, db=0, pwd=None, nodes=None, capacity=100000000, error_rate=0.00000001, key='task_filter'):
redis_client = queues.get_redis_client(redis_type, host, port, db, pwd, nodes)
if filter_type == 'set':
filter_client = filter.RedisFilter(conn=redis_client, key=key)
elif filter_type == 'bloom':
filter_client = filter.BloomFilter(conn=redis_client, capacity=capacity, error_rate=error_rate, key=key)
return filter_client
queue_client = queues.get_redis_client(QUEUE_REDIS_TYPE, QUEUE_REDIS_HOST, QUEUE_REDIS_PORT, QUEUE_REDIS_DB, QUEUE_REDIS_PWD, QUEUE_REDIS_NODES)
task_queues = queues.RedisQueues(conn=queue_client)
if IS_FILTER:
filter_client = get_filter(FILTER_TYPE, FILTER_REDIS_TYPE, FILTER_REDIS_HOST, FILTER_REDIS_PORT, FILTER_REDIS_DB, FILTER_REDIS_PWD,
from datetime import datetime
from decimal import Decimal
from xml.etree import ElementTree
import requests
from huey import RedisHuey, crontab
from models import ExchangeRates
huey = RedisHuey()
HISTORY_URL = 'https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist.xml'
LAST_90_URL = 'https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml'
@huey.periodic_task(crontab(minute='5', hour='*'))
def update_rates(history=False):
r = requests.get(HISTORY_URL if history else LAST_90_URL)
envelope = ElementTree.fromstring(r.content)
namespaces = {
'gesmes': 'http://www.gesmes.org/xml/2002-08-01',
'eurofxref': 'http://www.ecb.int/vocabulary/2002-08-01/eurofxref'
}
def huey(self):
if self._huey is None:
huey_config = self.huey_settings.copy()
name = huey_config.pop('name', self.default_queue_name())
conn_kwargs = huey_config.pop('connection', {})
# Don't need consumer or default settings here.
huey_config.pop('consumer', None)
huey_config.pop('default', None)
huey_config.setdefault('always_eager', settings.DEBUG)
huey_config.update(conn_kwargs)
self._huey = RedisHuey(name, **huey_config)
return self._huey