How to use the s3transfer.utils.OSUtils function in s3transfer

To help you get started, we’ve selected a few s3transfer examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github boto / s3transfer / tests / unit / View on Github external
class WriteCollector(object):
    """A utility to collect information about writes and seeks"""
    def __init__(self):
        self._pos = 0
        self.writes = []

    def seek(self, pos):
        self._pos = pos

    def write(self, data):
        self.writes.append((self._pos, data))
        self._pos += len(data)

class AlwaysIndicatesSpecialFileOSUtils(OSUtils):
    """OSUtil that always returns True for is_special_file"""
    def is_special_file(self, filename):
        return True

class CancelledStreamWrapper(object):
    """A wrapper to trigger a cancellation while stream reading

    Forces the transfer coordinator to cancel after a certain amount of reads
    :param stream: The underlying stream to read from
    :param transfer_coordinator: The coordinator for the transfer
    :param num_reads: On which read to sigal a cancellation. 0 is the first
    def __init__(self, stream, transfer_coordinator, num_reads=0):
        self._stream = stream
github boto / s3transfer / tests / unit / View on Github external
def test_open_file_chunk_reader_from_fileobj(self):
        with open(self.filename, 'rb') as f:
            reader = OSUtils().open_file_chunk_reader_from_fileobj(
                f, len(self.content), len(self.content), [self.callback])

            # The returned reader should be a ReadFileChunk.
            self.assertIsInstance(reader, ReadFileChunk)
            # The content of the reader should be correct.
            self.assertEqual(, self.content)
            # Callbacks should be disabled depspite being passed in.
            self.assertEqual(self.amounts_seen, [])
            self.assertEqual(self.num_close_callback_calls, 0)
github boto / s3transfer / tests / unit / View on Github external
def setUp(self):
        super(BaseDownloadOutputManagerTest, self).setUp()
        self.osutil = OSUtils()

        # Create a file to write to
        self.tempdir = tempfile.mkdtemp()
        self.filename = os.path.join(self.tempdir, 'myfile')

        self.call_args = CallArgs(fileobj=self.filename)
        self.future = self.get_transfer_future(self.call_args)
        self.io_executor = BoundedExecutor(1000, 1)
github boto / s3transfer / tests / unit / View on Github external
def test_rename_file(self):
        new_filename = os.path.join(self.tempdir, 'newfoo')
        OSUtils().rename_file(self.filename, new_filename)
github boto / s3transfer / tests / unit / View on Github external
from s3transfer.processpool import SHUTDOWN_SIGNAL
from s3transfer.processpool import ignore_ctrl_c
from s3transfer.processpool import DownloadFileRequest
from s3transfer.processpool import GetObjectJob
from s3transfer.processpool import ProcessTransferConfig
from s3transfer.processpool import ProcessPoolDownloader
from s3transfer.processpool import ProcessPoolTransferFuture
from s3transfer.processpool import ProcessPoolTransferMeta
from s3transfer.processpool import TransferMonitor
from s3transfer.processpool import TransferState
from s3transfer.processpool import ClientFactory
from s3transfer.processpool import GetObjectSubmitter
from s3transfer.processpool import GetObjectWorker

class RenameFailingOSUtils(OSUtils):
    def __init__(self, exception):
        self.exception = exception

    def rename_file(self, current_filename, new_filename):
        raise self.exception

class TestIgnoreCtrlC(unittest.TestCase):
    @skip_if_windows('os.kill() with SIGINT not supported on Windows')
    def test_ignore_ctrl_c(self):
        with ignore_ctrl_c():
                os.kill(os.getpid(), signal.SIGINT)
            except KeyboardInterrupt:
      'The ignore_ctrl_c context manager should have '
                          'ignored the KeyboardInterrupt exception')
github boto / s3transfer / tests / View on Github external
def setUp(self):
        super(BaseSubmissionTaskTest, self).setUp()
        self.config = TransferConfig()
        self.osutil = OSUtils()
        self.executor = BoundedExecutor(
                IN_MEMORY_UPLOAD_TAG: TaskSemaphore(10),
                IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(10)
github boto / s3transfer / tests / unit / View on Github external
def setUp(self):
        super(TestGetObjectSubmitter, self).setUp()
        self.transfer_config = ProcessTransferConfig()
        self.client_factory = mock.Mock(ClientFactory)
        self.client_factory.create_client.return_value = self.client
        self.transfer_monitor = TransferMonitor()
        self.osutil = mock.Mock(OSUtils)
        self.download_request_queue = queue.Queue()
        self.worker_queue = queue.Queue()
        self.submitter = GetObjectSubmitter(
        self.transfer_id = self.transfer_monitor.notify_new_transfer()
        self.bucket = 'bucket'
        self.key = 'key'
        self.filename = 'myfile'
        self.temp_filename = 'myfile.temp'
        self.osutil.get_temp_filename.return_value = self.temp_filename
github boto / s3transfer / tests / unit / View on Github external
def test_is_special_file_for_non_existant_file(self):
        non_existant_filename = os.path.join(self.tempdir, 'no-exist')
github quiltdata / quilt / api / python / quilt3 / View on Github external
def _upload_file(ctx, size, src_path, dest_bucket, dest_key):
    s3_client = ctx.s3_client

    if size < s3_transfer_config.multipart_threshold:
        with OSUtils().open_file_chunk_reader(src_path, 0, size, [ctx.progress]) as fd:
            resp = s3_client.put_object(

        version_id = resp.get('VersionId')  # Absent in unversioned buckets.
        ctx.done(make_s3_url(dest_bucket, dest_key, version_id))
        resp = s3_client.create_multipart_upload(
        upload_id = resp['UploadId']

        adjuster = ChunksizeAdjuster()
github boto / boto3 / boto3 / s3 / View on Github external
def __init__(self, client=None, config=None, osutil=None, manager=None):
        if not client and not manager:
            raise ValueError(
                'Either a boto3.Client or s3transfer.manager.TransferManager '
                'must be provided'
        if manager and any([client, config, osutil]):
            raise ValueError(
                'Manager cannot be provided with client, config, '
                'nor osutil. These parameters are mutually exclusive.'
        if config is None:
            config = TransferConfig()
        if osutil is None:
            osutil = OSUtils()
        if manager:
            self._manager = manager
            self._manager = create_transfer_manager(client, config, osutil)