How to use the omniduct.filesystems.base.FileSystemClient function in omniduct

To help you get started, we’ve selected a few omniduct examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github airbnb / omniduct / omniduct / filesystems / stub.py View on Github external
from omniduct.filesystems.base import FileSystemClient


class StubFsClient(FileSystemClient):

    PROTOCOLS = []
    DEFAULT_PORT = None

    def _init(self):
        pass

    # Connection management

    def _connect(self):
        raise NotImplementedError

    def _is_connected(self):
        raise NotImplementedError

    def _disconnect(self):
github airbnb / omniduct / omniduct / filesystems / local.py View on Github external
import datetime
import errno
import os
import shutil
import six
import sys
from io import open

from interface_meta import override

from .base import FileSystemClient, FileSystemFileDesc


class LocalFsClient(FileSystemClient):
    """
    `LocalFsClient` is a `Duct` that implements the `FileSystemClient` common
    API, and exposes the local filesystem.

    Unlike most other filesystems, `LocalFsClient` defaults to the current
    working directory on the local machine, rather than the home directory
    as used on remote filesystems. To change this, you can always execute:
    ```
    local_fs.path_cwd = local_fs.path_home
    ```
    """

    PROTOCOLS = ['localfs']

    @override
    def _init(self):
github airbnb / omniduct / omniduct / filesystems / webhdfs.py View on Github external
import random
from functools import partial

from interface_meta import override

from .base import FileSystemClient, FileSystemFileDesc
from .local import LocalFsClient

# Python 2 compatibility imports
try:
    FileNotFoundError
except NameError:
    FileNotFoundError = IOError


class WebHdfsClient(FileSystemClient):
    """
    This Duct connects to an Apache WebHDFS server using the `pywebhdfs` library.

    Attributes:
        namenodes (list): A list of hosts that are acting as namenodes for
            the HDFS cluster in form ":".
    """

    PROTOCOLS = ['webhdfs']
    DEFAULT_PORT = 50070

    @override
    def _init(self, namenodes=None, auto_conf=False, auto_conf_cluster=None,
              auto_conf_path=None, **kwargs):
        """
        namenodes (list): A list of hosts that are acting as namenodes for
github airbnb / omniduct / omniduct / remotes / base.py View on Github external
def __init__(self, smartcards=None, **kwargs):
        """
        Args:
            smartcards (dict): Mapping of smartcard names to system libraries
                compatible with `ssh-add -s '' ...`.
        """

        self.smartcards = smartcards
        self.__port_forwarding_register = PortForwardingRegister()

        FileSystemClient.__init_with_kwargs__(self, kwargs, port=self.DEFAULT_PORT)
github airbnb / omniduct / omniduct / filesystems / s3.py View on Github external
import logging

from interface_meta import override

from omniduct.filesystems.base import FileSystemClient, FileSystemFileDesc

# Python 2 compatibility imports
try:
    FileNotFoundError
except NameError:
    FileNotFoundError = IOError


class S3Client(FileSystemClient):
    """
    This Duct connects to an Amazon S3 bucket instance using the `boto3`
    library. Authentication is (optionally) handled using `opinel`.

    Attributes:
        bucket (str): The name of the Amazon S3 bucket to use.
        aws_profile (str): The name of configured AWS profile to use. This should
            refer to the name of a profile configured in, for example,
            `~/.aws/credentials`. Authentication is handled by the `opinel`
            library, which is also aware of environment variables.
    """

    PROTOCOLS = ['s3']
    DEFAULT_PORT = 80

    @override
github airbnb / omniduct / omniduct / caches / filesystem.py View on Github external
def _prepare(self):
        Cache._prepare(self)

        if self.registry is not None:
            if isinstance(self.fs, six.string_types):
                self.fs = self.registry.lookup(self.fs, kind=FileSystemCache.Type.FILESYSTEM)
        assert isinstance(self.fs, FileSystemClient), "Provided cache is not an instance of `omniduct.filesystems.base.FileSystemClient`."

        self._prepare_cache()
github airbnb / omniduct / omniduct / remotes / base.py View on Github external
def deregister(self, remote_host, remote_port):
        """
        Deregister a port-forward connection.

        Args:
            remote_host (str): The remote host.
            remote_port (int): The remote port.

        Returns:
            tuple: A tuple of local port and implementation-specific
                connection artifact, if it exists, and `None` otherwise.
        """
        return self._register.pop('{}:{}'.format(remote_host, remote_port))


class RemoteClient(FileSystemClient):
    """
    An abstract class providing the common API for all remote clients.

    Attributes:
        smartcard (dict): Mapping of smartcard names to system libraries
            compatible with `ssh-add -s '' ...`.
    """
    __doc_attrs = """
    smartcard (dict): Mapping of smartcard names to system libraries
        compatible with `ssh-add -s '' ...`.
    """

    DUCT_TYPE = Duct.Type.REMOTE
    DEFAULT_PORT = None

    @quirk_docs('_init', mro=True)