How to use the aiokafka.AIOKafkaConsumer function in aiokafka

To help you get started, we’ve selected a few aiokafka examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github CoinAlpha / hummingbot / wings / data_source / radar_relay_local_cluster_order_book_data_source.py View on Github external
async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
        """
        Listens to real-time order book diff messages from Radar Relay.
        """
        while True:
            try:
                consumer: AIOKafkaConsumer = AIOKafkaConsumer(self.DIFF_TOPIC_NAME,
                                                              loop=ev_loop,
                                                              bootstrap_servers=conf.kafka_2["bootstrap_servers"])
                await consumer.start()
                partition: TopicPartition = list(consumer.assignment())[0]
                await consumer.seek_to_end(partition)

                while True:
                    response: Dict[TopicPartition, List[ConsumerRecord]] = await consumer.getmany(partition,
                                                                                                  timeout_ms=1000)
                    if partition in response:
                        for record in response[partition]:
                            output.put_nowait(self.order_book_class.diff_message_from_kafka(record))
            except asyncio.CancelledError:
                raise
            except Exception:
                self.logger().error("Unknown error. Retrying after 5 seconds.", exc_info=True)
github CoinAlpha / hummingbot / wings / data_source / ddex_local_cluster_order_book_data_source.py View on Github external
async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
        """
        Listens to real-time order book snapshot messages from DDEX.
        """
        while True:
            try:
                consumer: AIOKafkaConsumer = AIOKafkaConsumer(self.SNAPSHOT_TOPIC_NAME,
                                                              loop=ev_loop,
                                                              bootstrap_servers=conf.kafka_2["bootstrap_servers"])
                await consumer.start()
                partition: TopicPartition = list(consumer.assignment())[0]
                await consumer.seek_to_end(partition)

                while True:
                    response: Dict[TopicPartition, List[ConsumerRecord]] = await consumer.getmany(partition,
                                                                                                  timeout_ms=1000)
                    if partition in response:
                        for record in response[partition]:
                            output.put_nowait(self.order_book_class.snapshot_message_from_kafka(record))
            except asyncio.CancelledError:
                raise
            except:
                self.logger().error("Unknown error. Retrying after 5 seconds.", exc_info=True)
github eventifyio / eventify / eventify / drivers / kafka.py View on Github external
async def onJoin(self):
        loop = asyncio.get_event_loop()

        for handler in self.handlers:
            # initialize handler
            handler_instance = handler()
            handler_instance.set_session(self)

            if hasattr(handler_instance, 'init'):
                await handler_instance.init()

            if hasattr(handler_instance, 'on_event'):
                self.log.debug("subscribing to topic %s", handler_instance.subscribe_topic)
                # Used with base handler defined subscribe_topic
                if handler_instance.subscribe_topic is not None:
                    consumer = AIOKafkaConsumer(
                        handler_instance.subscribe_topic,
                        bootstrap_servers=self.transport_host,
                        loop=loop
                    )
                    await consumer.start()
                    self.log.debug("subscribed to topic: %s", handler_instance.subscribe_topic)

                    try:
                        async for msg in consumer:
                            await handler_instance.on_event(msg.value)
                    finally:
                        await consumer.stop()
                else:
                    # Used with config.json defined topics
                    if self.subscribed_topics is not None:
                        consumer = AIOKafkaConsumer(
github CoinAlpha / hummingbot / wings / data_source / ddex_local_cluster_order_book_data_source.py View on Github external
async def listen_for_order_book_diffs(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
        """
        Listens to real-time order book diff messages from DDEX.
        """
        while True:
            try:
                consumer: AIOKafkaConsumer = AIOKafkaConsumer(self.DIFF_TOPIC_NAME,
                                                              loop=ev_loop,
                                                              bootstrap_servers=conf.kafka_2["bootstrap_servers"])
                await consumer.start()
                partition: TopicPartition = list(consumer.assignment())[0]
                await consumer.seek_to_end(partition)

                while True:
                    response: Dict[TopicPartition, List[ConsumerRecord]] = await consumer.getmany(partition,
                                                                                                  timeout_ms=1000)
                    if partition in response:
                        for record in response[partition]:
                            output.put_nowait(self.order_book_class.diff_message_from_kafka(record))
            except asyncio.CancelledError:
                raise
            except Exception:
                self.logger().error("Unknown error. Retrying after 5 seconds.", exc_info=True)
github CoinAlpha / hummingbot / wings / data_source / radar_relay_local_cluster_order_book_data_source.py View on Github external
async def listen_for_order_book_snapshots(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
        """
        Listens to real-time order book snapshot messages from Radar Relay.
        """
        while True:
            try:
                consumer: AIOKafkaConsumer = AIOKafkaConsumer(self.SNAPSHOT_TOPIC_NAME,
                                                              loop=ev_loop,
                                                              bootstrap_servers=conf.kafka_2["bootstrap_servers"])
                await consumer.start()
                partition: TopicPartition = list(consumer.assignment())[0]
                await consumer.seek_to_end(partition)

                while True:
                    response: Dict[TopicPartition, List[ConsumerRecord]] = await consumer.getmany(partition,
                                                                                                  timeout_ms=1000)
                    if partition in response:
                        for record in response[partition]:
                            output.put_nowait(self.order_book_class.snapshot_message_from_kafka(record))
            except asyncio.CancelledError:
                raise
            except:
                self.logger().error("Unknown error. Retrying after 5 seconds.", exc_info=True)
github eventifyio / eventify / eventify / drivers / kafka.py View on Github external
handler_instance.subscribe_topic,
                        bootstrap_servers=self.transport_host,
                        loop=loop
                    )
                    await consumer.start()
                    self.log.debug("subscribed to topic: %s", handler_instance.subscribe_topic)

                    try:
                        async for msg in consumer:
                            await handler_instance.on_event(msg.value)
                    finally:
                        await consumer.stop()
                else:
                    # Used with config.json defined topics
                    if self.subscribed_topics is not None:
                        consumer = AIOKafkaConsumer(
                            bootstrap_servers=self.transport_host,
                            loop=loop,
                            group_id='my-group'
                        )
                        await consumer.start()

                        # Subscribe to all topics
                        for topic in self.subscribed_topics:
                            consumer.subscribe(topic)

                        try:
                            async for msg in consumer:
                                value = msg.value.decode()
                                await handler_instance.on_event(value)
                        except Exception as error:
                            self.log.error("Consumer error. %s", error)
github AlecAivazis / graphql-over-kafka / nautilus / network / events / consumers / kafka.py View on Github external
def __init__(self):
        # a dictionary to keep the question/answer correlation ids
        self._request_handlers = {}
        # if there is no loop assigned
        if not self.loop:
            # use the current one
            self.loop = asyncio.get_event_loop()

        # a placeholder for the event consumer task
        self._consumer_task = None

        # create a consumer instance
        self._consumer = AIOKafkaConsumer(
            self.consumer_channel,
            loop=self.loop,
            bootstrap_servers=self.server,
            auto_offset_reset=self.initial_offset
        )
        self._producer = AIOKafkaProducer(loop=self.loop, bootstrap_servers=self.server)
github aiven / karapace / karapace / schema_reader.py View on Github external
def init_consumer(self):
        # Group not set on purpose, all consumers read the same data
        self.consumer = AIOKafkaConsumer(
            self.config["topic_name"],
            loop=self.loop,
            enable_auto_commit=False,
            api_version="1.0.0",
            bootstrap_servers=self.config["bootstrap_uri"],
            client_id=self.config["client_id"],
            security_protocol=self.config["security_protocol"],
            ssl_context=None if self.config["security_protocol"] != "SSL" else create_ssl_context(self.config),
            auto_offset_reset="earliest",
        )
github aio-libs / aiokafka / benchmark / simple_consume_bench.py View on Github external
async def bench_simple(self):
        topic = self._topic
        loop = asyncio.get_event_loop()

        consumer = AIOKafkaConsumer(
            topic, group_id="test_group", auto_offset_reset="earliest",
            enable_auto_commit=False,
            bootstrap_servers=self._bootstrap_servers,
            loop=loop)
        await consumer.start()

        # We start from after producer connect
        reporter_task = loop.create_task(self._stats_report(loop.time()))
        try:
            total_msgs = 0

            if not self._use_iter:
                while True:
                    msg_set = await consumer.getmany(timeout_ms=1000)
                    if not msg_set:
                        break