How to use s3transfer - 10 common examples

pass


class ExceptionSubmissionTask(SubmissionTask):
    def _submit(self, transfer_future, executor=None, tasks_to_submit=None,
                additional_callbacks=None, exception=TaskFailureException):
        if executor and tasks_to_submit:
            for task_to_submit in tasks_to_submit:
                self._transfer_coordinator.submit(executor, task_to_submit)
        if additional_callbacks:
            for callback in additional_callbacks:
                callback()
        raise exception()


class StatusRecordingTransferCoordinator(TransferCoordinator):
    def __init__(self, transfer_id=None):
        super(StatusRecordingTransferCoordinator, self).__init__(transfer_id)
        self.status_changes = [self._status]

    def set_status_to_queued(self):
        super(StatusRecordingTransferCoordinator, self).set_status_to_queued()
        self._record_status_change()

    def set_status_to_running(self):
        super(StatusRecordingTransferCoordinator, self).set_status_to_running()
        self._record_status_change()

    def _record_status_change(self):
        self.status_changes.append(self._status)

self.is_first = True

            def submit(self, function):
                future = super(FailedDownloadParts, self).submit(function)
                if self.is_first:
                    # This is the download_parts_thread.
                    future.set_exception(
                        Exception("fake download parts error"))
                    self.is_first = False
                return future

        client = mock.Mock()
        response_body = b'foobarbaz'
        client.get_object.return_value = {'Body': six.BytesIO(response_body)}

        downloader = MultipartDownloader(client, TransferConfig(),
                                         InMemoryOSLayer({}),
                                         FailedDownloadParts)
        with self.assertRaisesRegexp(Exception, "fake download parts error"):
            downloader.download_file('bucket', 'key', 'filename',
                                     len(response_body), {})

def setUp(self):
        self.transfer_coordinator = TransferCoordinator()

def return_call_args(*args, **kwargs):
    return args, kwargs


def raise_exception(exception):
    raise exception


def get_exc_info(exception):
    try:
        raise_exception(exception)
    except:
        return sys.exc_info()


class RecordingTransferCoordinator(TransferCoordinator):
    def __init__(self):
        self.all_transfer_futures_ever_associated = set()
        super(RecordingTransferCoordinator, self).__init__()

    def add_associated_future(self, future):
        self.all_transfer_futures_ever_associated.add(future)
        super(RecordingTransferCoordinator, self).add_associated_future(future)


class ReturnFooTask(Task):
    def _main(self, **kwargs):
        return 'foo'


class SleepTask(Task):
    def _main(self, sleep_time, **kwargs):

def setUp(self):
        super(BaseTaskTest, self).setUp()
        self.transfer_coordinator = TransferCoordinator()

def test_transfer_id(self):
        transfer_coordinator = TransferCoordinator(transfer_id=1)
        self.assertEqual(transfer_coordinator.transfer_id, 1)

def test_repr(self):
        transfer_coordinator = TransferCoordinator(transfer_id=1)
        self.assertEqual(
            repr(transfer_coordinator), 'TransferCoordinator(transfer_id=1)')

def setUp(self):
        self.leaky_bucket = mock.Mock(LeakyBucket)
        self.time_utils = mock.Mock(TimeUtils)
        self.tempdir = tempfile.mkdtemp()
        self.content = b'a' * 1024 * 1024
        self.filename = os.path.join(self.tempdir, 'myfile')
        with open(self.filename, 'wb') as f:
            f.write(self.content)
        self.coordinator = TransferCoordinator()

def test_uses_provided_osutil(self):
        osutil = RecordingOSUtils()
        # Use the recording os utility for the transfer manager
        self._manager = TransferManager(self.client, self.config, osutil)

        self.add_put_object_response_with_default_expected_params()

        future = self.manager.upload(self.filename, self.bucket, self.key)
        future.result()

        # The upload should have used the os utility. We check this by making
        # sure that the recorded opens are as expected.
        expected_opens = [(self.filename, 'rb')]
        self.assertEqual(osutil.open_records, expected_opens)

def test_error_in_context_manager_cancels_incomplete_transfers(self):
        # The purpose of this test is to make sure if an error is raised
        # in the body of the context manager, incomplete transfers will
        # be cancelled with value of the exception wrapped by a CancelledError

        # NOTE: The fact that delete() was chosen to test this is arbitrary
        # other than it is the easiet to set up for the stubber.
        # The specific operation is not important to the purpose of this test.
        num_transfers = 100
        futures = []
        ref_exception_msg = 'arbitrary exception'

        for _ in range(num_transfers):
            self.stubber.add_response('delete_object', {})

        manager = TransferManager(
            self.client,
            TransferConfig(
                max_request_concurrency=1, max_submission_concurrency=1)
        )
        try:
            with manager:
                for i in range(num_transfers):
                    futures.append(manager.delete('mybucket', 'mykey'))
                raise ArbitraryException(ref_exception_msg)
        except ArbitraryException:
            # At least one of the submitted futures should have been
            # cancelled.
            with self.assertRaisesRegexp(FatalError, ref_exception_msg):
                for future in futures:
                    future.result()