Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
self.osutil = OSUtils()
self.worker = GetObjectWorker(
queue=self.queue,
client_factory=self.client_factory,
transfer_monitor=self.transfer_monitor,
osutil=self.osutil
)
self.transfer_id = self.transfer_monitor.notify_new_transfer()
self.bucket = 'bucket'
self.key = 'key'
self.remote_contents = b'my content'
self.temp_filename = self.files.create_file('tempfile', '')
self.extra_args = {}
self.offset = 0
self.final_filename = self.files.full_path('final_filename')
self.stream = six.BytesIO(self.remote_contents)
self.transfer_monitor.notify_expected_jobs_to_complete(
self.transfer_id, 1000)
def add_get_responses(self):
chunksize = self.config.multipart_chunksize
for i in range(0, len(self.content), chunksize):
if i + chunksize > len(self.content):
stream = six.BytesIO(self.content[i:])
self.stubber.add_response('get_object', {'Body': stream})
else:
stream = six.BytesIO(self.content[i:i+chunksize])
self.stubber.add_response('get_object', {'Body': stream})
self.stubbed_client = self.manager.StubbedClient()
self.mock_client_factory = self.client_factory_patch.start()
self.mock_client_factory.\
return_value.create_client.return_value = self.stubbed_client
self.files = FileCreator()
self.config = ProcessTransferConfig(
max_request_processes=1
)
self.downloader = ProcessPoolDownloader(config=self.config)
self.bucket = 'mybucket'
self.key = 'mykey'
self.filename = self.files.full_path('filename')
self.remote_contents = b'my content'
self.stream = six.BytesIO(self.remote_contents)
def test_download_file_ranged_download(self):
half_of_content_length = int(len(self.remote_contents)/2)
self.stubbed_client.add_response(
'head_object', {'ContentLength': len(self.remote_contents)})
self.stubbed_client.add_response(
'get_object', {
'Body': six.BytesIO(
self.remote_contents[:half_of_content_length])}
)
self.stubbed_client.add_response(
'get_object', {
'Body': six.BytesIO(
self.remote_contents[half_of_content_length:])}
)
downloader = ProcessPoolDownloader(
config=ProcessTransferConfig(
multipart_chunksize=half_of_content_length,
multipart_threshold=half_of_content_length,
max_request_processes=1
)
)
with downloader:
downloader.download_file(self.bucket, self.key, self.filename)
self.assert_contents(self.filename, self.remote_contents)
def test_download_for_seekable_filelike_obj(self):
self.add_head_object_response()
self.add_successful_get_object_responses()
# Create a file-like object to test. In this case, it is a BytesIO
# object.
bytes_io = six.BytesIO()
future = self.manager.download(
self.bucket, self.key, bytes_io, self.extra_args)
future.result()
# Ensure that the contents are correct
bytes_io.seek(0)
self.assertEqual(self.content, bytes_io.read())
{
'method': 'head_object',
'service_response': {
'ContentLength': len(self.content)
}
},
{
'method': 'get_object',
'service_response': {
'Body': six.BytesIO(self.content[0:4])
}
},
{
'method': 'get_object',
'service_response': {
'Body': six.BytesIO(self.content[4:8])
}
},
{
'method': 'get_object',
'service_response': {
'Body': six.BytesIO(self.content[8:])
}
def open_nonseekable(self, filename, mode):
self.open_call_args.append((filename, mode))
return NonSeekableWriter(six.BytesIO(self.content))
def test_upload_for_seekable_filelike_obj_that_has_been_seeked(self):
self.add_put_object_response_with_default_expected_params()
bytes_io = six.BytesIO(self.content)
seek_pos = 5
bytes_io.seek(seek_pos)
future = self.manager.upload(
bytes_io, self.bucket, self.key, self.extra_args)
future.result()
self.assert_expected_client_calls_were_correct()
self.assertEqual(b''.join(self.sent_bodies), self.content[seek_pos:])
def test_upload_for_seekable_filelike_obj(self):
self.add_create_multipart_response_with_default_expected_params()
self.add_upload_part_responses_with_default_expected_params()
self.add_complete_multipart_response_with_default_expected_params()
bytes_io = six.BytesIO(self.content)
future = self.manager.upload(
bytes_io, self.bucket, self.key, self.extra_args)
future.result()
self.assert_expected_client_calls_were_correct()
self.assert_upload_part_bodies_were_correct()
def test_is_compatible_bytes_io(self):
self.assertTrue(
self.upload_input_manager.is_compatible(six.BytesIO()))