How to use the aiokafka.producer.AIOKafkaProducer function in aiokafka

To help you get started, we’ve selected a few aiokafka examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github aio-libs / aiokafka / tests / test_producer.py View on Github external
async def test_producer_send_batch(self):
        key = b'test key'
        value = b'test value'
        max_batch_size = 10000

        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts,
            max_batch_size=max_batch_size)
        await producer.start()

        partitions = await producer.partitions_for(self.topic)
        partition = partitions.pop()

        # silly method to find current offset for this partition
        resp = await producer.send_and_wait(
            self.topic, value=b'discovering offset', partition=partition)
        offset = resp.offset

        # only fills up to its limits, then returns None
        batch = producer.create_batch()
        self.assertEqual(batch.record_count(), 0)
        num = 0
github aio-libs / aiokafka / tests / test_transactional_producer.py View on Github external
async def test_producer_transactional_fences_off_previous(self):
        # Test 2 producers fencing one another by using the same
        # transactional_id

        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts,
            transactional_id="sobaka_producer", client_id="p1")
        await producer.start()
        self.add_cleanup(producer.stop)

        producer2 = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts,
            transactional_id="sobaka_producer", client_id="p2")
        await producer2.start()
        self.add_cleanup(producer2.stop)
        async with producer2.transaction():
            await producer2.send_and_wait(self.topic, b'hello, Kafka! 2')

        with self.assertRaises(ProducerFenced):
            async with producer.transaction():
                await producer.send_and_wait(self.topic, b'hello, Kafka!')
github aio-libs / aiokafka / tests / test_producer.py View on Github external
async def test_producer_start(self):
        with self.assertRaises(ValueError):
            producer = AIOKafkaProducer(loop=self.loop, acks=122)

        with self.assertRaises(ValueError):
            producer = AIOKafkaProducer(loop=self.loop, api_version="3.4.5")

        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        await producer.start()
        self.assertNotEqual(producer.client.api_version, 'auto')
        partitions = await producer.partitions_for('some_topic_name')
        self.assertEqual(len(partitions), 2)
        self.assertEqual(partitions, set([0, 1]))
        await producer.stop()
        self.assertEqual(producer._closed, True)
github aio-libs / aiokafka / tests / test_producer.py View on Github external
async def test_producer_send_error(self):
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts,
            retry_backoff_ms=100,
            linger_ms=5, request_timeout_ms=400)
        await producer.start()

        async def mocked_send(nodeid, req):
            # RequestTimedOutCode error for partition=0
            return ProduceResponse[0]([(self.topic, [(0, 7, 0), (1, 0, 111)])])

        with mock.patch.object(producer.client, 'send') as mocked:
            mocked.side_effect = mocked_send
            fut1 = await producer.send(self.topic, b'text1', partition=0)
            fut2 = await producer.send(self.topic, b'text2', partition=1)
            with self.assertRaises(RequestTimedOutError):
                await fut1
            resp = await fut2
github aio-libs / aiokafka / tests / test_transactional_consumer.py View on Github external
async def test_consumer_transactional_abort(self):
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts,
            transactional_id="sobaka_producer")
        await producer.start()
        self.add_cleanup(producer.stop)

        producer2 = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        await producer2.start()
        self.add_cleanup(producer2.stop)

        consumer = AIOKafkaConsumer(
            self.topic, loop=self.loop,
            bootstrap_servers=self.hosts,
            auto_offset_reset="earliest",
            isolation_level="read_committed")
        await consumer.start()
github aio-libs / aiokafka / tests / test_consumer.py View on Github external
async def test_consumer_wait_topic(self):
        topic = "some-test-topic-for-autocreate"
        consumer = AIOKafkaConsumer(
            topic, loop=self.loop, bootstrap_servers=self.hosts)
        await consumer.start()
        self.add_cleanup(consumer.stop)
        consume_task = self.loop.create_task(consumer.getone())
        # just to be sure getone does not fail (before produce)
        await asyncio.sleep(0.5, loop=self.loop)
        self.assertFalse(consume_task.done())

        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        await producer.start()
        await producer.send(topic, b'test msg')
        await producer.stop()

        data = await consume_task
        self.assertEqual(data.value, b'test msg')
github aio-libs / aiokafka / tests / test_producer.py View on Github external
def test_producer_indempotence_configuration(self):
        with self.assertRaises(ValueError):
            AIOKafkaProducer(
                loop=self.loop, acks=1, enable_idempotence=True)
        producer = AIOKafkaProducer(
            loop=self.loop, enable_idempotence=True)
        self.add_cleanup(producer.stop)
        self.assertEqual(producer._sender._acks, -1)  # -1 is set for `all`
        self.assertIsNotNone(producer._txn_manager)
github aio-libs / aiokafka / tests / test_producer.py View on Github external
async def test_producer_warn_unclosed(self):
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        producer_ref = weakref.ref(producer)
        await producer.start()

        with self.silence_loop_exception_handler():
            with self.assertWarnsRegex(
                    ResourceWarning, "Unclosed AIOKafkaProducer"):
                del producer
                gc.collect()
        # Assure that the reference was properly collected
        self.assertIsNone(producer_ref())
github aiven / karapace / karapace / schema_registry_apis.py View on Github external
def _create_producer(self):
        self.producer = AIOKafkaProducer(
            bootstrap_servers=self.config["bootstrap_uri"],
            security_protocol=self.config["security_protocol"],
            ssl_context=None if self.config["security_protocol"] != "SSL" else create_ssl_context(self.config),
            metadata_max_age_ms=self.config["metadata_max_age_ms"],
            loop=self.loop,
        )