Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
'class': 'logging.StreamHandler',
'formatter': 'default',
'stream': 'ext://sys.stdout'
},
'json': {
'class': 'c2cwsgiutils.pyramid_logging.JsonLogHandler',
'stream': 'ext://sys.stdout'
},
},
'formatters': {
'default': {
'format': format_
}
}
})
sentry.init()
def run(self):
auth_view(self.request)
if 'command' not in self.request.POST:
raise pyramid.httpexceptions.HTTPBadRequest("The POST argument 'command' is required")
command = shlex.split(self.request.POST['command'])
if command[0] not in self.gene.config.get('server', {}).get('allowed_commands', [
'generate_tiles', 'generate_controller'
]):
raise pyramid.httpexceptions.HTTPBadRequest(
"The given executable '{}' is not allowed".format(command[0])
)
lt = LogThread(command)
lt.start()
return pyramid.httpexceptions.HTTPFound(self.request.route_url('admin'))
def _time_iteration(self, generator, func_name):
while True:
timer = stats.timer()
try:
tile = next(generator)
except StopIteration:
break
except RuntimeError as e:
if isinstance(e.__cause__, StopIteration):
# since python 3.7, a StopIteration is wrapped in a RuntimeError (PEP 479)
break
else:
raise
timer.stop(self._get_stats_name(func_name, tile))
yield tile
def __call__(self, tile):
if tile is None:
logger.warning("The tile is None")
return None
if tile.error:
action = 'error'
elif tile.data:
action = 'create'
else:
action = 'delete'
layer = tile.metadata.get('layer', '- No layer -')
run = tile.metadata.get('run', -1)
with stats.timer_context(['db_logger', 'insert']):
with self.connection.cursor() as cursor:
try:
cursor.execute(
'INSERT INTO {} (layer, run, action, tile) '
'VALUES (%(layer)s, %(run)s, %(action)s::varchar(7), %(tile)s)'.
format(self.full_table),
{'layer': layer, 'action': action, 'tile': str(tile.tilecoord), 'run': run}
)
except psycopg2.IntegrityError:
self.connection.rollback()
cursor.execute(
'UPDATE {} SET action = %(action)s '
'WHERE layer = %(layer)s AND run = %(run)s AND tile = %(tile)s'.
format(self.full_table),
{'layer': layer, 'action': action, 'tile': str(tile.tilecoord), 'run': run}
)
def get_status(gene):
store = get_queue_store(gene.config, False)
kind = 'redis' if 'redis' in gene.config else 'sqs'
stats_prefix = [kind, gene.config[kind]['queue']]
with stats.timer_context(stats_prefix + ['get_stats']):
status = store.get_status()
return [name + ': ' + str(value) for name, value in status.items()]
def __call__(self, tile):
if tile:
stats.increment_counter(["tiles"])
return tile
def main():
stats.init_backends({})
parser = ArgumentParser(description='Used to generate the tiles', prog=sys.argv[0])
add_comon_options(parser, dimensions=True)
parser.add_argument(
'--get-hash', metavar="TILE",
help='get the empty tiles hash, use the specified TILE z/x/y'
)
parser.add_argument(
'--get-bbox', metavar="TILE",
help='get the bbox of a tile, use the specified TILE z/x/y, or z/x/y:+n/+n for metatiles'
)
parser.add_argument(
'--role', default='local', choices=('local', 'master', 'slave'),
help='local/master/slave, master to file the queue and '
'slave to generate the tiles'
)
parser.add_argument(
def put_one(self, tile):
with stats.timer_context(self._get_stats_name('put_one', tile)):
return self._tile_store.put_one(tile)
def list(self):
count = 0
while True:
queues = self._master.xreadgroup(
groupname=STREAM_GROUP,
consumername=CONSUMER_NAME,
streams={self._name: ">"},
count=1,
block=round(self._timeout_ms),
)
if not queues:
queues = self._claim_olds()
if queues is None:
stats.set_gauge(["redis", self._name_str, "nb_messages"], 0)
stats.set_gauge(["redis", self._name_str, "pending"], 0)
if queues is None and self._stop_if_empty:
break
if queues:
for redis_message in queues:
queue_name, queue_messages = redis_message
assert queue_name == self._name
for message in queue_messages:
id_, body = message
try:
tile = decode_message(body[b"message"], from_redis=True, sqs_message=id_)
yield tile
except Exception:
logger.warning("Failed decoding the Redis message", exc_info=True)
stats.increment_counter(["redis", self._name_str, "decode_error"])
count += 1
def callback(tile):
if tile:
if hasattr(tile, self.attr):
timer = getattr(tile, self.attr)
delta_t = timer.stop()
if statistics:
statistics.add(delta_t)
else:
setattr(tile, self.attr, stats.timer([key]))
return tile