Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
async def test_producer_send_batch(self):
key = b'test key'
value = b'test value'
max_batch_size = 10000
producer = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts,
max_batch_size=max_batch_size)
await producer.start()
partitions = await producer.partitions_for(self.topic)
partition = partitions.pop()
# silly method to find current offset for this partition
resp = await producer.send_and_wait(
self.topic, value=b'discovering offset', partition=partition)
offset = resp.offset
# only fills up to its limits, then returns None
batch = producer.create_batch()
self.assertEqual(batch.record_count(), 0)
num = 0
async def test_producer_transactional_fences_off_previous(self):
# Test 2 producers fencing one another by using the same
# transactional_id
producer = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts,
transactional_id="sobaka_producer", client_id="p1")
await producer.start()
self.add_cleanup(producer.stop)
producer2 = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts,
transactional_id="sobaka_producer", client_id="p2")
await producer2.start()
self.add_cleanup(producer2.stop)
async with producer2.transaction():
await producer2.send_and_wait(self.topic, b'hello, Kafka! 2')
with self.assertRaises(ProducerFenced):
async with producer.transaction():
await producer.send_and_wait(self.topic, b'hello, Kafka!')
async def test_producer_start(self):
with self.assertRaises(ValueError):
producer = AIOKafkaProducer(loop=self.loop, acks=122)
with self.assertRaises(ValueError):
producer = AIOKafkaProducer(loop=self.loop, api_version="3.4.5")
producer = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts)
await producer.start()
self.assertNotEqual(producer.client.api_version, 'auto')
partitions = await producer.partitions_for('some_topic_name')
self.assertEqual(len(partitions), 2)
self.assertEqual(partitions, set([0, 1]))
await producer.stop()
self.assertEqual(producer._closed, True)
async def test_producer_send_error(self):
producer = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts,
retry_backoff_ms=100,
linger_ms=5, request_timeout_ms=400)
await producer.start()
async def mocked_send(nodeid, req):
# RequestTimedOutCode error for partition=0
return ProduceResponse[0]([(self.topic, [(0, 7, 0), (1, 0, 111)])])
with mock.patch.object(producer.client, 'send') as mocked:
mocked.side_effect = mocked_send
fut1 = await producer.send(self.topic, b'text1', partition=0)
fut2 = await producer.send(self.topic, b'text2', partition=1)
with self.assertRaises(RequestTimedOutError):
await fut1
resp = await fut2
async def test_consumer_transactional_abort(self):
producer = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts,
transactional_id="sobaka_producer")
await producer.start()
self.add_cleanup(producer.stop)
producer2 = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts)
await producer2.start()
self.add_cleanup(producer2.stop)
consumer = AIOKafkaConsumer(
self.topic, loop=self.loop,
bootstrap_servers=self.hosts,
auto_offset_reset="earliest",
isolation_level="read_committed")
await consumer.start()
async def test_consumer_wait_topic(self):
topic = "some-test-topic-for-autocreate"
consumer = AIOKafkaConsumer(
topic, loop=self.loop, bootstrap_servers=self.hosts)
await consumer.start()
self.add_cleanup(consumer.stop)
consume_task = self.loop.create_task(consumer.getone())
# just to be sure getone does not fail (before produce)
await asyncio.sleep(0.5, loop=self.loop)
self.assertFalse(consume_task.done())
producer = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts)
await producer.start()
await producer.send(topic, b'test msg')
await producer.stop()
data = await consume_task
self.assertEqual(data.value, b'test msg')
def test_producer_indempotence_configuration(self):
with self.assertRaises(ValueError):
AIOKafkaProducer(
loop=self.loop, acks=1, enable_idempotence=True)
producer = AIOKafkaProducer(
loop=self.loop, enable_idempotence=True)
self.add_cleanup(producer.stop)
self.assertEqual(producer._sender._acks, -1) # -1 is set for `all`
self.assertIsNotNone(producer._txn_manager)
async def test_producer_warn_unclosed(self):
producer = AIOKafkaProducer(
loop=self.loop, bootstrap_servers=self.hosts)
producer_ref = weakref.ref(producer)
await producer.start()
with self.silence_loop_exception_handler():
with self.assertWarnsRegex(
ResourceWarning, "Unclosed AIOKafkaProducer"):
del producer
gc.collect()
# Assure that the reference was properly collected
self.assertIsNone(producer_ref())
def _create_producer(self):
self.producer = AIOKafkaProducer(
bootstrap_servers=self.config["bootstrap_uri"],
security_protocol=self.config["security_protocol"],
ssl_context=None if self.config["security_protocol"] != "SSL" else create_ssl_context(self.config),
metadata_max_age_ms=self.config["metadata_max_age_ms"],
loop=self.loop,
)