Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
"""
Listens to real-time order book diff messages from Radar Relay.
"""
while True:
try:
consumer: AIOKafkaConsumer = AIOKafkaConsumer(self.DIFF_TOPIC_NAME,
loop=ev_loop,
bootstrap_servers=conf.kafka_2["bootstrap_servers"])
await consumer.start()
partition: TopicPartition = list(consumer.assignment())[0]
await consumer.seek_to_end(partition)
while True:
response: Dict[TopicPartition, List[ConsumerRecord]] = await consumer.getmany(partition,
timeout_ms=1000)
if partition in response:
for record in response[partition]:
output.put_nowait(self.order_book_class.diff_message_from_kafka(record))
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unknown error. Retrying after 5 seconds.", exc_info=True)
async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
"""
Listens to real-time order book snapshot messages from DDEX.
"""
while True:
try:
consumer: AIOKafkaConsumer = AIOKafkaConsumer(self.SNAPSHOT_TOPIC_NAME,
loop=ev_loop,
bootstrap_servers=conf.kafka_2["bootstrap_servers"])
await consumer.start()
partition: TopicPartition = list(consumer.assignment())[0]
await consumer.seek_to_end(partition)
while True:
response: Dict[TopicPartition, List[ConsumerRecord]] = await consumer.getmany(partition,
timeout_ms=1000)
if partition in response:
for record in response[partition]:
output.put_nowait(self.order_book_class.snapshot_message_from_kafka(record))
except asyncio.CancelledError:
raise
except:
self.logger().error("Unknown error. Retrying after 5 seconds.", exc_info=True)
async def onJoin(self):
loop = asyncio.get_event_loop()
for handler in self.handlers:
# initialize handler
handler_instance = handler()
handler_instance.set_session(self)
if hasattr(handler_instance, 'init'):
await handler_instance.init()
if hasattr(handler_instance, 'on_event'):
self.log.debug("subscribing to topic %s", handler_instance.subscribe_topic)
# Used with base handler defined subscribe_topic
if handler_instance.subscribe_topic is not None:
consumer = AIOKafkaConsumer(
handler_instance.subscribe_topic,
bootstrap_servers=self.transport_host,
loop=loop
)
await consumer.start()
self.log.debug("subscribed to topic: %s", handler_instance.subscribe_topic)
try:
async for msg in consumer:
await handler_instance.on_event(msg.value)
finally:
await consumer.stop()
else:
# Used with config.json defined topics
if self.subscribed_topics is not None:
consumer = AIOKafkaConsumer(
async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
"""
Listens to real-time order book diff messages from DDEX.
"""
while True:
try:
consumer: AIOKafkaConsumer = AIOKafkaConsumer(self.DIFF_TOPIC_NAME,
loop=ev_loop,
bootstrap_servers=conf.kafka_2["bootstrap_servers"])
await consumer.start()
partition: TopicPartition = list(consumer.assignment())[0]
await consumer.seek_to_end(partition)
while True:
response: Dict[TopicPartition, List[ConsumerRecord]] = await consumer.getmany(partition,
timeout_ms=1000)
if partition in response:
for record in response[partition]:
output.put_nowait(self.order_book_class.diff_message_from_kafka(record))
except asyncio.CancelledError:
raise
except Exception:
self.logger().error("Unknown error. Retrying after 5 seconds.", exc_info=True)
async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
"""
Listens to real-time order book snapshot messages from Radar Relay.
"""
while True:
try:
consumer: AIOKafkaConsumer = AIOKafkaConsumer(self.SNAPSHOT_TOPIC_NAME,
loop=ev_loop,
bootstrap_servers=conf.kafka_2["bootstrap_servers"])
await consumer.start()
partition: TopicPartition = list(consumer.assignment())[0]
await consumer.seek_to_end(partition)
while True:
response: Dict[TopicPartition, List[ConsumerRecord]] = await consumer.getmany(partition,
timeout_ms=1000)
if partition in response:
for record in response[partition]:
output.put_nowait(self.order_book_class.snapshot_message_from_kafka(record))
except asyncio.CancelledError:
raise
except:
self.logger().error("Unknown error. Retrying after 5 seconds.", exc_info=True)
handler_instance.subscribe_topic,
bootstrap_servers=self.transport_host,
loop=loop
)
await consumer.start()
self.log.debug("subscribed to topic: %s", handler_instance.subscribe_topic)
try:
async for msg in consumer:
await handler_instance.on_event(msg.value)
finally:
await consumer.stop()
else:
# Used with config.json defined topics
if self.subscribed_topics is not None:
consumer = AIOKafkaConsumer(
bootstrap_servers=self.transport_host,
loop=loop,
group_id='my-group'
)
await consumer.start()
# Subscribe to all topics
for topic in self.subscribed_topics:
consumer.subscribe(topic)
try:
async for msg in consumer:
value = msg.value.decode()
await handler_instance.on_event(value)
except Exception as error:
self.log.error("Consumer error. %s", error)
def __init__(self):
# a dictionary to keep the question/answer correlation ids
self._request_handlers = {}
# if there is no loop assigned
if not self.loop:
# use the current one
self.loop = asyncio.get_event_loop()
# a placeholder for the event consumer task
self._consumer_task = None
# create a consumer instance
self._consumer = AIOKafkaConsumer(
self.consumer_channel,
loop=self.loop,
bootstrap_servers=self.server,
auto_offset_reset=self.initial_offset
)
self._producer = AIOKafkaProducer(loop=self.loop, bootstrap_servers=self.server)
def init_consumer(self):
# Group not set on purpose, all consumers read the same data
self.consumer = AIOKafkaConsumer(
self.config["topic_name"],
loop=self.loop,
enable_auto_commit=False,
api_version="1.0.0",
bootstrap_servers=self.config["bootstrap_uri"],
client_id=self.config["client_id"],
security_protocol=self.config["security_protocol"],
ssl_context=None if self.config["security_protocol"] != "SSL" else create_ssl_context(self.config),
auto_offset_reset="earliest",
)
async def bench_simple(self):
topic = self._topic
loop = asyncio.get_event_loop()
consumer = AIOKafkaConsumer(
topic, group_id="test_group", auto_offset_reset="earliest",
enable_auto_commit=False,
bootstrap_servers=self._bootstrap_servers,
loop=loop)
await consumer.start()
# We start from after producer connect
reporter_task = loop.create_task(self._stats_report(loop.time()))
try:
total_msgs = 0
if not self._use_iter:
while True:
msg_set = await consumer.getmany(timeout_ms=1000)
if not msg_set:
break