Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
groupname=STREAM_GROUP,
consumername=CONSUMER_NAME,
min_idle_time=self._pending_timeout_ms,
message_ids=to_drop,
)
drop_ids = [drop_message[0] for drop_message in drop_messages]
self._master.xack(self._name, STREAM_GROUP, *drop_ids)
self._master.xdel(self._name, *drop_ids)
for _, drop_message in drop_messages:
tile = decode_message(drop_message[b"message"])
self._master.xadd(
name=self._errors_name,
fields=dict(tilecoord=str(tile.tilecoord)),
maxlen=self._max_errors_nb,
)
stats.increment_counter(["redis", self._name_str, "dropped"], len(to_drop))
if to_steal:
messages = self._master.xclaim(
name=self._name,
groupname=STREAM_GROUP,
consumername=CONSUMER_NAME,
min_idle_time=self._pending_timeout_ms,
message_ids=to_steal,
)
stats.increment_counter(["redis", self._name_str, "stolen"], len(to_steal))
return [[self._name, messages]]
else:
# Empty means there are pending jobs, but they are not old enough to be stolen
return []
def add_stats(tile):
if tile and tile.error:
stats.increment_counter(['error', tile.metadata.get('layer', 'None')])
return tile
self.imap(add_stats)
def __call__(self, tile):
if tile and tile.error:
stats.increment_counter(["errors"])
return tile
stats.set_gauge(["redis", self._name_str, "nb_messages"], 0)
stats.set_gauge(["redis", self._name_str, "pending"], 0)
if queues is None and self._stop_if_empty:
break
if queues:
for redis_message in queues:
queue_name, queue_messages = redis_message
assert queue_name == self._name
for message in queue_messages:
id_, body = message
try:
tile = decode_message(body[b"message"], from_redis=True, sqs_message=id_)
yield tile
except Exception:
logger.warning("Failed decoding the Redis message", exc_info=True)
stats.increment_counter(["redis", self._name_str, "decode_error"])
count += 1
if count % 10 == 0:
stats.set_gauge(
["redis", self._name_str, "nb_messages"], self._slave.xlen(name=self._name)
)
pending = self._slave.xpending(self._name, STREAM_GROUP)
stats.set_gauge(["redis", self._name_str, "pending"], pending["pending"])
self._master.xadd(
name=self._errors_name,
fields=dict(tilecoord=str(tile.tilecoord)),
maxlen=self._max_errors_nb,
)
stats.increment_counter(["redis", self._name_str, "dropped"], len(to_drop))
if to_steal:
messages = self._master.xclaim(
name=self._name,
groupname=STREAM_GROUP,
consumername=CONSUMER_NAME,
min_idle_time=self._pending_timeout_ms,
message_ids=to_steal,
)
stats.increment_counter(["redis", self._name_str, "stolen"], len(to_steal))
return [[self._name, messages]]
else:
# Empty means there are pending jobs, but they are not old enough to be stolen
return []